dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/test/helpers.rb

@@ -0,0 +1,134 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/element'
+require 'fluent/msgpack_factory'
+require 'fluent/time'
+
+module Fluent
+  module Test
+    module Helpers
+      # See "Example Custom Assertion: https://test-unit.github.io/test-unit/en/Test/Unit/Assertions.html
+      def assert_equal_event_time(expected, actual, message = nil)
+        expected_s = "#{Time.at(expected.sec)} (nsec #{expected.nsec})"
+        actual_s   = "#{Time.at(actual.sec)  } (nsec #{actual.nsec})"
+        message = build_message(message, <<EOT, expected_s, actual_s)
+<?> expected but was
+<?>.
+EOT
+        assert_block(message) do
+          expected.is_a?(Fluent::EventTime) && actual.is_a?(Fluent::EventTime) && expected.sec == actual.sec && expected.nsec == actual.nsec
+        end
+      end
+
+      def config_element(name = 'test', argument = '', params = {}, elements = [])
+        Fluent::Config::Element.new(name, argument, params, elements)
+      end
+
+      def event_time(str=nil, format: nil)
+        if str
+          if format
+            Fluent::EventTime.from_time(Time.strptime(str, format))
+          else
+            Fluent::EventTime.parse(str)
+          end
+        else
+          Fluent::EventTime.now
+        end
+      end
+
+      def with_timezone(tz)
+        oldtz, ENV['TZ'] = ENV['TZ'], tz
+        yield
+      ensure
+        ENV['TZ'] = oldtz
+      end
+
+      def with_worker_config(root_dir: nil, workers: nil, worker_id: nil, &block)
+        if workers
+          if worker_id
+            if worker_id >= workers
+              raise "worker_id must be between 0 and (workers - 1)"
+            end
+          else
+            worker_id = 0
+          end
+        end
+
+        opts = {}
+        opts['root_dir'] = root_dir if root_dir
+        opts['workers'] = workers if workers
+
+        ENV['SERVERENGINE_WORKER_ID'] = worker_id.to_s
+        Fluent::SystemConfig.overwrite_system_config(opts, &block)
+      ensure
+        ENV.delete('SERVERENGINE_WORKER_ID')
+      end
+
+      def time2str(time, localtime: false, format: nil)
+        if format
+          if localtime
+            Time.at(time).strftime(format)
+          else
+            Time.at(time).utc.strftime(format)
+          end
+        else
+          if localtime
+            Time.at(time).iso8601
+          else
+            Time.at(time).utc.iso8601
+          end
+        end
+      end
+
+      def msgpack(type)
+        case type
+        when :factory
+          Fluent::MessagePackFactory.factory
+        when :packer
+          Fluent::MessagePackFactory.packer
+        when :unpacker
+          Fluent::MessagePackFactory.unpacker
+        else
+          raise ArgumentError, "unknown msgpack object type '#{type}'"
+        end
+      end
+
+      #
+      # Use this method with v0.12 compatibility layer.
+      #
+      # For v0.14 API, use `driver.logs` instead.
+      #
+      def capture_log(driver)
+        tmp = driver.instance.log.out
+        driver.instance.log.out = StringIO.new
+        yield
+        return driver.instance.log.out.string
+      ensure
+        driver.instance.log.out = tmp
+      end
+
+      def capture_stdout
+        out = StringIO.new
+        $stdout = out
+        yield
+        out.string.force_encoding('utf-8')
+      ensure
+        $stdout = STDOUT
+      end
+    end
+  end
+end

data/lib/fluent/test/input_test.rb

@@ -0,0 +1,174 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/engine'
+require 'fluent/time'
+require 'fluent/test/base'
+
+module Fluent
+  module Test
+    class InputTestDriver < TestDriver
+      def initialize(klass, &block)
+        super(klass, &block)
+        @emit_streams = []
+        @event_streams = []
+        @expects = nil
+        # for checking only the number of emitted records during run
+        @expected_emits_length = nil
+        @run_timeout = 5
+        @run_post_conditions = []
+      end
+
+      def expect_emit(tag, time, record)
+        (@expects ||= []) << [tag, time, record]
+        self
+      end
+
+      def expected_emits
+        @expects ||= []
+      end
+
+      attr_accessor :expected_emits_length
+      attr_accessor :run_timeout
+      attr_reader :emit_streams, :event_streams
+
+      def emits
+        all = []
+        @emit_streams.each {|tag,events|
+          events.each {|time,record|
+            all << [tag, time, record]
+          }
+        }
+        all
+      end
+
+      def events
+        all = []
+        @emit_streams.each {|tag,events|
+          all.concat events
+        }
+        all
+      end
+
+      def records
+        all = []
+        @emit_streams.each {|tag,events|
+          events.each {|time,record|
+            all << record
+          }
+        }
+        all
+      end
+
+      def register_run_post_condition(&block)
+        if block
+          @run_post_conditions << block
+        end
+      end
+
+      def register_run_breaking_condition(&block)
+        if block
+          @run_breaking_conditions ||= []
+          @run_breaking_conditions << block
+        end
+      end
+
+      def run_should_stop?
+        # Should stop running if post conditions are not registered.
+        return true unless @run_post_conditions
+
+        # Should stop running if all of the post conditions are true.
+        return true if @run_post_conditions.all? {|proc| proc.call }
+
+        # Should stop running if any of the breaking conditions is true.
+        # In this case, some post conditions may be not true.
+        return true if @run_breaking_conditions && @run_breaking_conditions.any? {|proc| proc.call }
+
+        false
+      end
+
+      module EmitStreamWrapper
+        def emit_stream_callee=(method)
+          @emit_stream_callee = method
+        end
+        def emit_stream(tag, es)
+          @emit_stream_callee.call(tag, es)
+        end
+      end
+
+      def run(num_waits = 10, &block)
+        m = method(:emit_stream)
+        unless Engine.singleton_class.ancestors.include?(EmitStreamWrapper)
+          Engine.singleton_class.prepend EmitStreamWrapper
+        end
+        Engine.emit_stream_callee = m
+        unless instance.router.singleton_class.ancestors.include?(EmitStreamWrapper)
+          instance.router.singleton_class.prepend EmitStreamWrapper
+        end
+        instance.router.emit_stream_callee = m
+
+        super(num_waits) {
+          block.call if block
+
+          if @expected_emits_length || @expects || @run_post_conditions
+            # counters for emits and emit_streams
+            i, j = 0, 0
+
+            # Events of expected length will be emitted at the end.
+            max_length = @expected_emits_length
+            max_length ||= @expects.length if @expects
+            if max_length
+              register_run_post_condition do
+                i == max_length
+              end
+            end
+
+            # Set running timeout to avoid infinite loop caused by some errors.
+            started_at = Time.now
+            register_run_breaking_condition do
+              Time.now >= started_at + @run_timeout
+            end
+
+            until run_should_stop?
+              if j >= @emit_streams.length
+                sleep 0.01
+                next
+              end
+
+              tag, events = @emit_streams[j]
+              events.each do |time, record|
+                if @expects
+                  assert_equal(@expects[i], [tag, time, record])
+                  assert_equal_event_time(@expects[i][1], time) if @expects[i][1].is_a?(Fluent::EventTime)
+                end
+                i += 1
+              end
+              j += 1
+            end
+            assert_equal(@expects.length, i) if @expects
+          end
+        }
+        self
+      end
+
+      private
+      def emit_stream(tag, es)
+        @event_streams << es
+        @emit_streams << [tag, es.to_a]
+      end
+    end
+  end
+end

data/lib/fluent/test/log.rb

@@ -0,0 +1,79 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'serverengine'
+require 'fluent/log'
+
+module Fluent
+  module Test
+    class DummyLogDevice
+      attr_reader :logs
+      attr_accessor :flush_logs
+
+      def initialize
+        @logs = []
+        @flush_logs = true
+        @use_stderr = false
+      end
+
+      def reset
+        @logs = [] if @flush_logs
+      end
+
+      def tty?
+        false
+      end
+
+      def puts(*args)
+        args.each{ |arg| write(arg + "\n") }
+      end
+
+      def write(message)
+        if @use_stderr
+          STDERR.write message
+        end
+        @logs.push message
+      end
+
+      def flush
+        true
+      end
+
+      def close
+        true
+      end
+    end
+
+    class TestLogger < Fluent::PluginLogger
+      def initialize
+        @logdev = DummyLogDevice.new
+        dl_opts = {}
+        dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO
+        logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
+        log = Fluent::Log.new(logger)
+        super(log)
+      end
+
+      def reset
+        @logdev.reset
+      end
+
+      def logs
+        @logdev.logs
+      end
+    end
+  end
+end

data/lib/fluent/test/output_test.rb

@@ -0,0 +1,156 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/engine'
+require 'fluent/event'
+require 'fluent/test/input_test'
+
+module Fluent
+  module Test
+    class TestOutputChain
+      def initialize
+        @called = 0
+      end
+
+      def next
+        @called += 1
+      end
+
+      attr_reader :called
+    end
+
+
+    class OutputTestDriver < InputTestDriver
+      def initialize(klass, tag='test', &block)
+        super(klass, &block)
+        @tag = tag
+      end
+
+      attr_accessor :tag
+
+      def emit(record, time=EventTime.now)
+        es = OneEventStream.new(time, record)
+        @instance.emit_events(@tag, es)
+      end
+    end
+
+
+    class BufferedOutputTestDriver < InputTestDriver
+      def initialize(klass, tag='test', &block)
+        super(klass, &block)
+        @entries = []
+        @expected_buffer = nil
+        @tag = tag
+
+        def @instance.buffer
+          @buffer
+        end
+      end
+
+      attr_accessor :tag
+
+      def emit(record, time=EventTime.now)
+        @entries << [time, record]
+        self
+      end
+
+      def expect_format(str)
+        (@expected_buffer ||= '') << str
+      end
+
+      def run(num_waits = 10, &block)
+        result = nil
+        super(num_waits) {
+          block.call if block
+
+          es = ArrayEventStream.new(@entries)
+          buffer = @instance.format_stream(@tag, es)
+
+          if @expected_buffer
+            assert_equal(@expected_buffer, buffer)
+          end
+
+          chunk = if @instance.instance_eval{ @chunk_key_tag }
+                    @instance.buffer.generate_chunk(@instance.metadata(@tag, nil, nil)).staged!
+                  else
+                    @instance.buffer.generate_chunk(@instance.metadata(nil, nil, nil)).staged!
+                  end
+          chunk.concat(buffer, es.size)
+
+          begin
+            result = @instance.write(chunk)
+          ensure
+            chunk.purge
+          end
+        }
+        result
+      end
+    end
+
+    class TimeSlicedOutputTestDriver < InputTestDriver
+      def initialize(klass, tag='test', &block)
+        super(klass, &block)
+        @entries = []
+        @expected_buffer = nil
+        @tag = tag
+      end
+
+      attr_accessor :tag
+
+      def emit(record, time=EventTime.now)
+        @entries << [time, record]
+        self
+      end
+
+      def expect_format(str)
+        (@expected_buffer ||= '') << str
+      end
+
+      def run(&block)
+        result = []
+        super {
+          block.call if block
+
+          buffer = ''
+          lines = {}
+          # v0.12 TimeSlicedOutput doesn't call #format_stream
+          @entries.each do |time, record|
+            meta = @instance.metadata(@tag, time, record)
+            line = @instance.format(@tag, time, record)
+            buffer << line
+            lines[meta] ||= []
+            lines[meta] << line
+          end
+
+          if @expected_buffer
+            assert_equal(@expected_buffer, buffer)
+          end
+
+          lines.keys.each do |meta|
+            chunk = @instance.buffer.generate_chunk(meta).staged!
+            chunk.append(lines[meta])
+            begin
+              result.push(@instance.write(chunk))
+            ensure
+              chunk.purge
+            end
+          end
+        }
+        result
+      end
+    end
+  end
+end

data/lib/fluent/test/parser_test.rb

@@ -0,0 +1,70 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/parser'
+require 'fluent/config'
+
+module Fluent
+  module Test
+    class ParserTestDriver
+      def initialize(klass_or_str, format=nil, conf={}, &block)
+        if klass_or_str.is_a?(Class)
+          if block
+            # Create new class for test w/ overwritten methods
+            #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass_or_str.name
+            klass_or_str = Class.new(klass_or_str)
+            klass_or_str.define_singleton_method("name") { klass_name }
+            klass_or_str.module_eval(&block)
+          end
+          case klass_or_str.instance_method(:initialize).arity
+          when 0
+            @instance = klass_or_str.new
+          when -2
+            # for RegexpParser
+            @instance = klass_or_str.new(format, conf)
+          end
+        elsif klass_or_str.is_a?(String)
+          @instance = Fluent::Plugin.new_parser(klass_or_str)
+        else
+          @instance = klass_or_str
+        end
+        @config = Config.new
+      end
+
+      attr_reader :instance, :config
+
+      def configure(conf)
+        case conf
+        when Fluent::Config::Element
+          @config = conf
+        when String
+          @config = Config.parse(conf, 'fluent.conf')
+        when Hash
+          @config = Config::Element.new('ROOT', '', conf, [])
+        else
+          raise "Unknown type... #{conf}"
+        end
+        @instance.configure(@config)
+        self
+      end
+
+      def parse(text, &block)
+        @instance.parse(text, &block)
+      end
+    end
+  end
+end

data/lib/fluent/test/startup_shutdown.rb

@@ -0,0 +1,46 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'serverengine'
+require 'fileutils'
+
+module Fluent
+  module Test
+    module StartupShutdown
+      def startup
+        socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      end
+
+      def shutdown
+        @server.close
+      end
+
+      def self.setup
+        @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        @server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+      end
+
+      def self.teardown
+        @server.close
+        # on Windows, socket_manager_path is a TCP port number
+        FileUtils.rm_f @socket_manager_path unless Fluent.windows?
+      end
+    end
+  end
+end