fluentd 0.12.40 → 0.14.0

data/lib/fluent/system_config.rb

@@ -18,21 +18,6 @@ require 'fluent/configurable'
 require 'fluent/config/element'
 
 module Fluent
-  module SystemConfigMixin
-    def system_config
-      @_system_config || Fluent::Engine.system_config
-    end
-
-    def system_config_override(opts={})
-      unless @_system_config
-        @_system_config = Fluent::Engine.system_config.dup
-      end
-      opts.each_pair do |key, value|
-        @_system_config.send(:"#{key.to_s}=", value)
-      end
-    end
-  end
-
   class SystemConfig
     include Configurable
 
@@ -46,11 +31,15 @@ module Fluent
     config_param :rpc_endpoint, :string, default: nil
     config_param :enable_get_dump, :bool, default: nil
     config_param :process_name, default: nil
+    config_param :file_permission, default: nil do |v|
+      v.to_i(8)
+    end
+    config_param :dir_permission, default: nil do |v|
+      v.to_i(8)
+    end
 
     def self.create(conf)
-      systems = conf.elements.select { |e|
-        e.name == 'system'
-      }
+      systems = conf.elements(name: 'system')
       return SystemConfig.new if systems.empty?
       raise Fluent::ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
 
@@ -61,6 +50,16 @@ module Fluent
       Fluent::Config::Element.new('<SYSTEM>', '', {}, [])
     end
 
+    def self.overwrite_system_config(hash)
+      older = defined?($_system_config) ? $_system_config : nil
+      begin
+        $_system_config = SystemConfig.new(Fluent::Config::Element.new('system', '', hash, []))
+        yield
+      ensure
+        $_system_config = older
+      end
+    end
+
     def initialize(conf=nil)
       super()
       conf ||= SystemConfig.blank_system_config
@@ -77,6 +76,8 @@ module Fluent
       s.rpc_endpoint = @rpc_endpoint
       s.enable_get_dump = @enable_get_dump
       s.process_name = @process_name
+      s.file_permission = @file_permission
+      s.dir_permission = @dir_permission
 
       s
     end
@@ -92,7 +93,30 @@ module Fluent
         @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
         @enable_get_dump = system.enable_get_dump unless system.enable_get_dump.nil?
         @process_name = system.process_name unless system.process_name.nil?
+        @file_permission = system.file_permission unless system.file_permission.nil?
+        @dir_permission = system.dir_permission unless system.dir_permission.nil?
       }
     end
+
+    module Mixin
+      def system_config
+        require 'fluent/engine'
+        unless defined?($_system_config)
+          $_system_config = nil
+        end
+        (instance_variable_defined?("@_system_config") && @_system_config) ||
+          $_system_config || Fluent::Engine.system_config
+      end
+
+      def system_config_override(opts={})
+        require 'fluent/engine'
+        if !instance_variable_defined?("@_system_config") || @_system_config.nil?
+          @_system_config = (defined?($_system_config) && $_system_config ? $_system_config : Fluent::Engine.system_config).dup
+        end
+        opts.each_pair do |key, value|
+          @_system_config.send(:"#{key.to_s}=", value)
+        end
+      end
+    end
   end
 end

data/lib/fluent/test.rb

@@ -22,5 +22,10 @@ require 'fluent/test/output_test'
 require 'fluent/test/filter_test'
 require 'fluent/test/parser_test'
 require 'fluent/test/formatter_test'
+require 'serverengine'
 
-$log ||= Fluent::Log.new(Fluent::Test::DummyLogDevice.new)
+dl_opts = {}
+dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO
+logdev = Fluent::Test::DummyLogDevice.new
+logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+$log ||= Fluent::Log.new(logger)

data/lib/fluent/test/base.rb

@@ -17,6 +17,7 @@
 require 'fluent/engine'
 require 'fluent/system_config'
 require 'fluent/config'
+require 'serverengine'
 
 module Fluent
   module Test
@@ -25,10 +26,10 @@ module Fluent
       engine = Fluent.const_set(:Engine, EngineClass.new).init(SystemConfig.new)
 
       engine.define_singleton_method(:now=) {|n|
-        @now = n.to_i
+        @now = n
       }
       engine.define_singleton_method(:now) {
-        @now || super()
+        @now ||= super()
       }
 
       nil
@@ -91,6 +92,10 @@ module Fluent
         @logs = []
       end
 
+      def reset
+        @logs = []
+      end
+
       def tty?
         false
       end
@@ -115,7 +120,15 @@ module Fluent
     class TestLogger < Fluent::PluginLogger
       def initialize
         @logdev = DummyLogDevice.new
-        super(Fluent::Log.new(@logdev))
+        dl_opts = {}
+        dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO
+        logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
+        log = Fluent::Log.new(logger)
+        super(log)
+      end
+
+      def reset
+        @logdev.reset
       end
 
       def logs
@@ -124,3 +137,10 @@ module Fluent
     end
   end
 end
+
+Test::Unit::Assertions.module_eval do
+  def assert_equal_event_time(a, b)
+    assert_equal(a.sec, b.sec)
+    assert_equal(a.nsec, b.nsec)
+  end
+end

data/lib/fluent/test/driver/base.rb

@@ -0,0 +1,247 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/test_event_router'
+
+require 'timeout'
+
+module Fluent
+  module Test
+    module Driver
+      class Base
+        def initialize(klass, opts: {}, &block)
+          if klass.is_a?(Class)
+            if block
+              # Create new class for test w/ overwritten methods
+              #   klass.dup is worse because its ancestors does NOT include original class name
+              klass = Class.new(klass)
+              klass.module_eval(&block)
+            end
+            @instance = klass.new
+          else
+            @instance = klass
+          end
+          if opts
+            @instance.system_config_override(opts)
+          end
+          @instance.log = TestLogger.new
+          @logs = @instance.log.out.logs
+
+          @run_post_conditions = []
+          @run_breaking_conditions = []
+
+          @broken = false
+
+          @event_streams = nil
+          @error_events = nil
+        end
+
+        attr_reader :instance, :logs
+
+        def configure(conf, syntax: :v1)
+          if conf.is_a?(Fluent::Config::Element)
+            @config = conf
+          else
+            @config = Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
+          end
+
+          if @instance.respond_to?(:router=)
+            @event_streams = []
+            @error_events = []
+
+            driver = self
+            mojule = Module.new do
+              define_method(:event_emitter_router) do |label_name|
+                TestEventRouter.new(driver)
+              end
+            end
+            @instance.singleton_class.module_eval do
+              prepend mojule
+            end
+          end
+
+          @instance.configure(@config)
+          self
+        end
+
+        def end_if(&block)
+          raise ArgumentError, "block is not given" unless block_given?
+          @run_post_conditions << block
+        end
+
+        def break_if(&block)
+          raise ArgumentError, "block is not given" unless block_given?
+          @run_breaking_conditions << block
+        end
+
+        def broken?
+          @broken
+        end
+
+        Emit = Struct.new(:tag, :es)
+        ErrorEvent = Struct.new(:tag, :time, :record, :error)
+
+        # via TestEventRouter
+        def emit_event_stream(tag, es)
+          @event_streams << Emit.new(tag, es)
+        end
+
+        def emit_error_event(tag, time, record, error)
+          @error_events << ErrorEvent.new(tag, time, record, error)
+        end
+
+        def events(tag: nil)
+          selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }
+          if block_given?
+            selected.each do |e|
+              e.es.each do |time, record|
+                yield e.tag, time, record
+              end
+            end
+          else
+            list = []
+            selected.each do |e|
+              e.es.each do |time, record|
+                list << [e.tag, time, record]
+              end
+            end
+            list
+          end
+        end
+
+        def error_events(tag: nil)
+          selected = @error_events.select{|e| tag.nil? ? true : e.tag == tag }
+          if block_given?
+            selected.each do |e|
+              yield e.tag, e.time, e.record, e.error
+            end
+          else
+            selected.map{|e| [e.tag, e.time, e.record, e.error] }
+          end
+        end
+
+        def run(expect_emits: nil, expect_records: nil, timeout: nil, start: true, shutdown: true, &block)
+          instance_start if start
+
+          if @instance.respond_to?(:thread_wait_until_start)
+            @instance.thread_wait_until_start
+          end
+          if @instance.respond_to?(:event_loop_wait_until_start)
+            @instance.event_loop_wait_until_start
+          end
+
+          begin
+            run_actual(expect_emits: expect_emits, expect_records: expect_records, timeout: timeout, &block)
+          ensure
+            instance_shutdown if shutdown
+          end
+        end
+
+        def instance_start
+          unless @instance.started?
+            @instance.start
+            instance_hook_after_started
+          end
+        end
+
+        def instance_hook_after_started
+          # insert hooks for tests available after instance.start
+        end
+
+        def instance_shutdown
+          @instance.stop            unless @instance.stopped?
+          @instance.before_shutdown unless @instance.before_shutdown?
+          @instance.shutdown        unless @instance.shutdown?
+
+          if @instance.respond_to?(:event_loop_wait_until_stop)
+            @instance.event_loop_wait_until_stop
+          end
+
+          @instance.after_shutdown  unless @instance.after_shutdown?
+          @instance.close     unless @instance.closed?
+
+          if @instance.respond_to?(:thread_wait_until_stop)
+            @instance.thread_wait_until_stop
+          end
+
+          @instance.terminate unless @instance.terminated?
+        end
+
+        def run_actual(expect_emits: nil, expect_records: nil, timeout: nil, &block)
+          if @instance.respond_to?(:_threads)
+            until @instance._threads.values.all?(&:alive?)
+              sleep 0.01
+            end
+          end
+
+          if @instance.respond_to?(:event_loop_running?)
+            until @instance.event_loop_running?
+              sleep 0.01
+            end
+          end
+
+          if expect_emits
+            @run_post_conditions << ->(){ @emit_streams.size >= expect_emits }
+          end
+          if expect_records
+            @run_post_conditions << ->(){ @emit_streams.reduce(0){|a, e| a + e.es.size } >= expected_records }
+          end
+          if timeout
+            stop_at = Time.now + timeout
+            @run_breaking_conditions << ->(){ Time.now >= stop_at }
+          end
+
+          if !block_given? && @run_post_conditions.empty? && @run_breaking_conditions.empty?
+            raise ArgumentError, "no stop conditions nor block specified"
+          end
+
+          if !block_given?
+            block = ->(){ sleep(0.1) until stop? }
+          end
+
+          if timeout
+            begin
+              Timeout.timeout(timeout * 1.1) do |sec|
+                block.call
+              end
+            rescue Timeout::Error
+              @broken = true
+            end
+          else
+            block.call
+          end
+        end
+
+        def stop?
+          # Should stop running if post conditions are not registered.
+          return true unless @run_post_conditions
+
+          # Should stop running if all of the post conditions are true.
+          return true if @run_post_conditions.all? {|proc| proc.call }
+
+          # Should stop running if some of the breaking conditions is true.
+          # In this case, some post conditions may be not true.
+          if @run_breaking_conditions.any? {|proc| proc.call }
+            @broken = true
+            return true
+          end
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/event_feeder.rb

@@ -0,0 +1,98 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+require 'fluent/time'
+
+module Fluent
+  module Test
+    module Driver
+      module EventFeeder
+        def initialize(klass, opts: {}, &block)
+          super
+          @default_tag = nil
+          @feed_method = nil
+        end
+
+        def run(default_tag: nil, **kwargs, &block)
+          @feed_method = if @instance.respond_to?(:filter_stream)
+                           :filter_stream
+                         else
+                           :emit_events
+                         end
+          if default_tag
+            @default_tag = default_tag
+          end
+          super(**kwargs, &block)
+        end
+
+        def feed_to_plugin(tag, es)
+          @instance.__send__(@feed_method, tag, es)
+        end
+
+        # d.run do
+        #   d.feed('tag', time, {record})
+        #   d.feed('tag', [ [time, {record}], [time, {record}], ... ])
+        #   d.feed('tag', es)
+        # end
+        # d.run(default_tag: 'tag') do
+        #   d.feed({record})
+        #   d.feed(time, {record})
+        #   d.feed([ [time, {record}], [time, {record}], ... ])
+        #   d.feed(es)
+        # end
+        def feed(*args)
+          case args.size
+          when 1
+            raise ArgumentError, "tag not specified without default_tag" unless @default_tag
+            case args.first
+            when Fluent::EventStream
+              feed_to_plugin(@default_tag, args.first)
+            when Array
+              feed_to_plugin(@default_tag, ArrayEventStream.new(args.first))
+            when Hash
+              record = args.first
+              time = Fluent::EventTime.now
+              feed_to_plugin(@default_tag, OneEventStream.new(time, record))
+            else
+              raise ArgumentError, "unexpected events object (neither event(Hash), EventStream nor Array): #{args.first.class}"
+            end
+          when 2
+            if args[0].is_a?(String) && (args[1].is_a?(Array) || args[1].is_a?(Fluent::EventStream))
+              tag, es = args
+              es = ArrayEventStream.new(es) if es.is_a?(Array)
+              feed_to_plugin(tag, es)
+            elsif @default_tag && (args[0].is_a?(Fluent::EventTime) || args[0].is_a?(Integer)) && args[1].is_a?(Hash)
+              time, record = args
+              feed_to_plugin(@default_tag, OneEventStream.new(time, record))
+            else
+              raise ArgumentError, "unexpected values of argument: #{args[0].class}, #{args[1].class}"
+            end
+          when 3
+            tag, time, record = args
+            if tag.is_a?(String) && (time.is_a?(Fluent::EventTime) || time.is_a?(Integer)) && record.is_a?(Hash)
+              feed_to_plugin(tag, OneEventStream.new(time, record))
+            else
+              raise ArgumentError, "unexpected values of argument: #{tag.class}, #{time.class}, #{record.class}"
+            end
+          else
+            raise ArgumentError, "unexpected number of arguments: #{args}"
+          end
+        end
+      end
+    end
+  end
+end