fluentd 0.12.40 → 1.6.2

data/lib/fluent/system_config.rb

@@ -18,39 +18,63 @@ require 'fluent/configurable'
 require 'fluent/config/element'
 
 module Fluent
-  module SystemConfigMixin
-    def system_config
-      @_system_config || Fluent::Engine.system_config
-    end
-
-    def system_config_override(opts={})
-      unless @_system_config
-        @_system_config = Fluent::Engine.system_config.dup
-      end
-      opts.each_pair do |key, value|
-        @_system_config.send(:"#{key.to_s}=", value)
-      end
-    end
-  end
-
   class SystemConfig
     include Configurable
 
-    config_param :log_level, default: nil do |level|
-      Log.str_to_level(level)
-    end
+    SYSTEM_CONFIG_PARAMETERS = [
+      :workers, :root_dir, :log_level,
+      :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
+      :log_event_verbose,
+      :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
+      :file_permission, :dir_permission, :counter_server, :counter_client,
+    ]
+
+    config_param :workers,   :integer, default: 1
+    config_param :root_dir,  :string, default: nil
+    config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: nil
     config_param :suppress_repeated_stacktrace, :bool, default: nil
-    config_param :emit_error_log_interval, :time, default: nil
+    config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
-    config_param :without_source, :bool, default: nil
-    config_param :rpc_endpoint, :string, default: nil
+    config_param :log_event_verbose,    :bool, default: nil
+    config_param :without_source,  :bool, default: nil
+    config_param :rpc_endpoint,    :string, default: nil
     config_param :enable_get_dump, :bool, default: nil
-    config_param :process_name, default: nil
+    config_param :process_name,    :string, default: nil
+    config_param :file_permission, default: nil do |v|
+      v.to_i(8)
+    end
+    config_param :dir_permission, default: nil do |v|
+      v.to_i(8)
+    end
+    config_section :log, required: false, init: true, multi: false do
+      config_param :format, :enum, list: [:text, :json], default: :text
+      config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
+    end
+
+    config_section :counter_server, multi: false do
+      desc 'scope name of counter server'
+      config_param :scope, :string
+
+      desc 'the port of counter server to listen to'
+      config_param :port, :integer, default: nil
+      desc 'the bind address of counter server to listen to'
+      config_param :bind, :string, default: nil
+
+      desc 'backup file path of counter values'
+      config_param :backup_path, :string
+    end
+
+    config_section :counter_client, multi: false do
+      desc 'the port of counter server'
+      config_param :port, :integer, default: nil
+      desc 'the IP address or hostname of counter server'
+      config_param :host, :string
+      desc 'the timeout of each operation'
+      config_param :timeout, :time, default: nil
+    end
 
     def self.create(conf)
-      systems = conf.elements.select { |e|
-        e.name == 'system'
-      }
+      systems = conf.elements(name: 'system')
       return SystemConfig.new if systems.empty?
       raise Fluent::ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
 
@@ -61,38 +85,103 @@ module Fluent
       Fluent::Config::Element.new('<SYSTEM>', '', {}, [])
     end
 
+    def self.overwrite_system_config(hash)
+      older = defined?($_system_config) ? $_system_config : nil
+      begin
+        $_system_config = SystemConfig.new(Fluent::Config::Element.new('system', '', hash, []))
+        yield
+      ensure
+        $_system_config = older
+      end
+    end
+
     def initialize(conf=nil)
       super()
       conf ||= SystemConfig.blank_system_config
       configure(conf)
     end
 
+    def configure(conf)
+      super
+
+      @log_level = Log.str_to_level(@log_level.to_s) if @log_level
+    end
+
     def dup
       s = SystemConfig.new
-      s.log_level = @log_level
-      s.suppress_repeated_stacktrace = @suppress_repeated_stacktrace
-      s.emit_error_log_interval = @emit_error_log_interval
-      s.suppress_config_dump = @suppress_config_dump
-      s.without_source = @without_source
-      s.rpc_endpoint = @rpc_endpoint
-      s.enable_get_dump = @enable_get_dump
-      s.process_name = @process_name
-
+      SYSTEM_CONFIG_PARAMETERS.each do |param|
+        s.__send__("#{param}=", instance_variable_get("@#{param}"))
+      end
       s
     end
 
+    def attach(supervisor)
+      system = self
+      supervisor.instance_eval {
+        SYSTEM_CONFIG_PARAMETERS.each do |param|
+          case param
+          when :rpc_endpoint, :enable_get_dump, :process_name, :file_permission, :dir_permission, :counter_server, :counter_client
+            next # doesn't exist in command line options
+          when :emit_error_log_interval
+            system.emit_error_log_interval = @suppress_interval if @suppress_interval
+          when :log_level
+            ll_value = instance_variable_get("@log_level")
+            # info level can't be specified via command line option.
+            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
+            if ll_value != Fluent::Log::LEVEL_INFO
+              system.log_level = ll_value
+            end
+          else
+            next unless instance_variable_defined?("@#{param}")
+            supervisor_value = instance_variable_get("@#{param}")
+            next if supervisor_value.nil? # it's not configured by command line options
+
+            system.send("#{param}=", supervisor_value)
+          end
+        end
+      }
+    end
+
     def apply(supervisor)
       system = self
       supervisor.instance_eval {
-        @log.level = @log_level = system.log_level unless system.log_level.nil?
-        @suppress_interval = system.emit_error_log_interval unless system.emit_error_log_interval.nil?
-        @suppress_config_dump = system.suppress_config_dump unless system.suppress_config_dump.nil?
-        @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
-        @without_source = system.without_source unless system.without_source.nil?
-        @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
-        @enable_get_dump = system.enable_get_dump unless system.enable_get_dump.nil?
-        @process_name = system.process_name unless system.process_name.nil?
+        SYSTEM_CONFIG_PARAMETERS.each do |param|
+          param_value = system.__send__(param)
+          next if param_value.nil?
+
+          case param
+          when :log_level
+            @log.level = @log_level = param_value
+          when :emit_error_log_interval
+            @suppress_interval = param_value
+          else
+            instance_variable_set("@#{param}", param_value)
+          end
+        end
+        #@counter_server = system.counter_server unless system.counter_server.nil?
+        #@counter_client = system.counter_client unless system.counter_client.nil?
       }
     end
+
+    module Mixin
+      def system_config
+        require 'fluent/engine'
+        unless defined?($_system_config)
+          $_system_config = nil
+        end
+        (instance_variable_defined?("@_system_config") && @_system_config) ||
+          $_system_config || Fluent::Engine.system_config
+      end
+
+      def system_config_override(opts={})
+        require 'fluent/engine'
+        if !instance_variable_defined?("@_system_config") || @_system_config.nil?
+          @_system_config = (defined?($_system_config) && $_system_config ? $_system_config : Fluent::Engine.system_config).dup
+        end
+        opts.each_pair do |key, value|
+          @_system_config.send(:"#{key.to_s}=", value)
+        end
+      end
+    end
   end
 end

data/lib/fluent/test/base.rb

@@ -14,26 +14,14 @@
 #    limitations under the License.
 #
 
+require 'fluent/config'
 require 'fluent/engine'
 require 'fluent/system_config'
-require 'fluent/config'
+require 'fluent/test/log'
+require 'serverengine'
 
 module Fluent
   module Test
-    def self.setup
-      Fluent.__send__(:remove_const, :Engine)
-      engine = Fluent.const_set(:Engine, EngineClass.new).init(SystemConfig.new)
-
-      engine.define_singleton_method(:now=) {|n|
-        @now = n.to_i
-      }
-      engine.define_singleton_method(:now) {
-        @now || super()
-      }
-
-      nil
-    end
-
     class TestDriver
       include ::Test::Unit::Assertions
 
@@ -42,7 +30,9 @@ module Fluent
           if block
             # Create new class for test w/ overwritten methods
             #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass.name
             klass = Class.new(klass)
+            klass.define_singleton_method("name") { klass_name }
             klass.module_eval(&block)
           end
           @instance = klass.new
@@ -74,6 +64,7 @@ module Fluent
       # num_waits is for checking thread status. This will be removed after improved plugin API
       def run(num_waits = 10, &block)
         @instance.start
+        @instance.after_start
         begin
           # wait until thread starts
           num_waits.times { sleep 0.05 }
@@ -83,44 +74,5 @@ module Fluent
         end
       end
     end
-
-    class DummyLogDevice
-      attr_reader :logs
-
-      def initialize
-        @logs = []
-      end
-
-      def tty?
-        false
-      end
-
-      def puts(*args)
-        args.each{ |arg| write(arg + "\n") }
-      end
-
-      def write(message)
-        @logs.push message
-      end
-
-      def flush
-        true
-      end
-
-      def close
-        true
-      end
-    end
-
-    class TestLogger < Fluent::PluginLogger
-      def initialize
-        @logdev = DummyLogDevice.new
-        super(Fluent::Log.new(@logdev))
-      end
-
-      def logs
-        @logdev.logs
-      end
-    end
   end
 end

data/lib/fluent/test/driver/base.rb

@@ -0,0 +1,224 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/config/element'
+require 'fluent/log'
+require 'fluent/clock'
+
+require 'serverengine/socket_manager'
+require 'fileutils'
+require 'timeout'
+
+module Fluent
+  module Test
+    module Driver
+      class TestTimedOut < RuntimeError; end
+
+      class Base
+        attr_reader :instance, :logs
+
+        DEFAULT_TIMEOUT = 300
+
+        def initialize(klass, opts: {}, &block)
+          if klass.is_a?(Class)
+            @instance = klass.new
+            if block
+              @instance.singleton_class.module_eval(&block)
+              @instance.send(:initialize)
+            end
+          else
+            @instance = klass
+          end
+          @instance.under_plugin_development = true
+
+          @socket_manager_server = nil
+
+          @logs = []
+
+          @run_post_conditions = []
+          @run_breaking_conditions = []
+          @broken = false
+        end
+
+        def configure(conf, syntax: :v1)
+          raise NotImplementedError
+        end
+
+        def end_if(&block)
+          raise ArgumentError, "block is not given" unless block_given?
+          @run_post_conditions << block
+        end
+
+        def break_if(&block)
+          raise ArgumentError, "block is not given" unless block_given?
+          @run_breaking_conditions << block
+        end
+
+        def broken?
+          @broken
+        end
+
+        def run(timeout: nil, start: true, shutdown: true, &block)
+          instance_start if start
+
+          timeout ||= DEFAULT_TIMEOUT
+          stop_at = Fluent::Clock.now + timeout
+          @run_breaking_conditions << ->(){ Fluent::Clock.now >= stop_at }
+
+          if !block_given? && @run_post_conditions.empty? && @run_breaking_conditions.empty?
+            raise ArgumentError, "no stop conditions nor block specified"
+          end
+
+          sleep_with_watching_threads = ->(){
+            if @instance.respond_to?(:_threads)
+              @instance._threads.values.each{|t| t.join(0) }
+            end
+            sleep 0.1
+          }
+
+          begin
+            retval = run_actual(timeout: timeout, &block)
+            sleep_with_watching_threads.call until stop?
+            retval
+          ensure
+            instance_shutdown if shutdown
+          end
+        end
+
+        def instance_start
+          if @instance.respond_to?(:server_wait_until_start)
+            @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+            if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+              FileUtils.rm_f @socket_manager_path
+            end
+            @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
+            ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+          end
+
+          unless @instance.started?
+            @instance.start
+          end
+          unless @instance.after_started?
+            @instance.after_start
+          end
+
+          if @instance.respond_to?(:thread_wait_until_start)
+            @instance.thread_wait_until_start
+          end
+
+          if @instance.respond_to?(:event_loop_wait_until_start)
+            @instance.event_loop_wait_until_start
+          end
+
+          instance_hook_after_started
+        end
+
+        def instance_hook_after_started
+          # insert hooks for tests available after instance.start
+        end
+
+        def instance_hook_before_stopped
+          # same with above
+        end
+
+        def instance_shutdown
+          instance_hook_before_stopped
+
+          show_errors_if_exists = ->(label, block){
+            begin
+              block.call
+            rescue => e
+              puts "unexpected error while #{label}, #{e.class}:#{e.message}"
+              e.backtrace.each do |bt|
+                puts "\t#{bt}"
+              end
+            end
+          }
+
+          show_errors_if_exists.call(:stop,            ->(){ @instance.stop unless @instance.stopped? })
+          show_errors_if_exists.call(:before_shutdown, ->(){ @instance.before_shutdown unless @instance.before_shutdown? })
+          show_errors_if_exists.call(:shutdown,        ->(){ @instance.shutdown unless @instance.shutdown? })
+          show_errors_if_exists.call(:after_shutdown,  ->(){ @instance.after_shutdown unless @instance.after_shutdown? })
+
+          if @instance.respond_to?(:server_wait_until_stop)
+            @instance.server_wait_until_stop
+          end
+
+          if @instance.respond_to?(:event_loop_wait_until_stop)
+            @instance.event_loop_wait_until_stop
+          end
+
+          show_errors_if_exists.call(:close, ->(){ @instance.close unless @instance.closed? })
+
+          if @instance.respond_to?(:thread_wait_until_stop)
+            @instance.thread_wait_until_stop
+          end
+
+          show_errors_if_exists.call(:terminate, ->(){ @instance.terminate unless @instance.terminated? })
+
+          if @socket_manager_server
+            @socket_manager_server.close
+            if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+              FileUtils.rm_f @socket_manager_path
+            end
+          end
+        end
+
+        def run_actual(timeout: DEFAULT_TIMEOUT, &block)
+          if @instance.respond_to?(:_threads)
+            sleep 0.1 until @instance._threads.values.all?(&:alive?)
+          end
+
+          if @instance.respond_to?(:event_loop_running?)
+            sleep 0.1 until @instance.event_loop_running?
+          end
+
+          if @instance.respond_to?(:_child_process_processes)
+            sleep 0.1 until @instance._child_process_processes.values.all?{|pinfo| pinfo.alive }
+          end
+
+          return_value = nil
+          begin
+            Timeout.timeout(timeout * 2) do |sec|
+              return_value = block.call if block_given?
+            end
+          rescue Timeout::Error
+            raise TestTimedOut, "Test case timed out with hard limit."
+          end
+          return_value
+        end
+
+        def stop?
+          # Should stop running if post conditions are not registered.
+          return true unless @run_post_conditions || @run_post_conditions.empty?
+
+          # Should stop running if all of the post conditions are true.
+          return true if @run_post_conditions.all? {|proc| proc.call }
+
+          # Should stop running if some of the breaking conditions is true.
+          # In this case, some post conditions may be not true.
+          if @run_breaking_conditions.any? {|proc| proc.call }
+            @broken = true
+            return true
+          end
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/base_owned.rb

@@ -0,0 +1,70 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base'
+
+require 'fluent/plugin/base'
+require 'fluent/plugin_id'
+require 'fluent/log'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Test
+    module Driver
+      class OwnerDummy < Fluent::Plugin::Base
+        include PluginId
+        include PluginLoggerMixin
+        include PluginHelper::Mixin
+      end
+
+      class BaseOwned < Base
+        attr_accessor :section_name
+
+        def initialize(klass, opts: {}, &block)
+          super
+
+          owner = OwnerDummy.new
+          if opts
+            owner.system_config_override(opts)
+          end
+          owner.log = TestLogger.new
+
+          if @instance.respond_to?(:owner=)
+            @instance.owner = owner
+            if opts
+              @instance.system_config_override(opts)
+            end
+          end
+
+          @logs = owner.log.out.logs
+          @section_name = ''
+        end
+
+        def configure(conf, syntax: :v1)
+          if conf.is_a?(Fluent::Config::Element)
+            @config = conf
+          elsif conf.is_a?(Hash)
+            @config = Fluent::Config::Element.new(@section_name, "", Hash[conf.map{|k,v| [k.to_s, v]}], [])
+          else
+            @config = Fluent::Config.parse(conf, @section_name, "", syntax: syntax)
+          end
+          @instance.configure(@config)
+          self
+        end
+      end
+    end
+  end
+end