fluentd 0.12.40 → 1.6.2

data/lib/fluent/test/helpers.rb

@@ -15,22 +15,22 @@
 #
 
 require 'fluent/config/element'
-require 'time'
+require 'fluent/msgpack_factory'
 require 'fluent/time'
 
 module Fluent
   module Test
     module Helpers
-      # See "Example Custom Assertion: http://test-unit.github.io/test-unit/en/Test/Unit/Assertions.html
+      # See "Example Custom Assertion: https://test-unit.github.io/test-unit/en/Test/Unit/Assertions.html
       def assert_equal_event_time(expected, actual, message = nil)
-        expected_s = "#{expected}"
-        actual_s   = "#{actual}"
+        expected_s = "#{Time.at(expected.sec)} (nsec #{expected.nsec})"
+        actual_s   = "#{Time.at(actual.sec)  } (nsec #{actual.nsec})"
         message = build_message(message, <<EOT, expected_s, actual_s)
-<?> time expected but was
+<?> expected but was
 <?>.
 EOT
         assert_block(message) do
-          expected.is_a?(Integer) && actual.is_a?(Integer) && expected == actual
+          expected.is_a?(Fluent::EventTime) && actual.is_a?(Fluent::EventTime) && expected.sec == actual.sec && expected.nsec == actual.nsec
         end
       end
 
@@ -38,15 +38,15 @@ EOT
         Fluent::Config::Element.new(name, argument, params, elements)
       end
 
-      def event_time(str = nil, format: nil)
+      def event_time(str=nil, format: nil)
         if str
           if format
-            Time.strptime(str, format).to_i
+            Fluent::EventTime.from_time(Time.strptime(str, format))
           else
-            Time.parse(str).to_i
+            Fluent::EventTime.parse(str)
           end
         else
-          Time.now.to_i
+          Fluent::EventTime.now
         end
       end
 
@@ -57,6 +57,27 @@ EOT
         ENV['TZ'] = oldtz
       end
 
+      def with_worker_config(root_dir: nil, workers: nil, worker_id: nil, &block)
+        if workers
+          if worker_id
+            if worker_id >= workers
+              raise "worker_id must be between 0 and (workers - 1)"
+            end
+          else
+            worker_id = 0
+          end
+        end
+
+        opts = {}
+        opts['root_dir'] = root_dir if root_dir
+        opts['workers'] = workers if workers
+
+        ENV['SERVERENGINE_WORKER_ID'] = worker_id.to_s
+        Fluent::SystemConfig.overwrite_system_config(opts, &block)
+      ensure
+        ENV.delete('SERVERENGINE_WORKER_ID')
+      end
+
       def time2str(time, localtime: false, format: nil)
         if format
           if localtime
@@ -73,6 +94,24 @@ EOT
         end
       end
 
+      def msgpack(type)
+        case type
+        when :factory
+          Fluent::MessagePackFactory.factory
+        when :packer
+          Fluent::MessagePackFactory.packer
+        when :unpacker
+          Fluent::MessagePackFactory.unpacker
+        else
+          raise ArgumentError, "unknown msgpack object type '#{type}'"
+        end
+      end
+
+      #
+      # Use this method with v0.12 compatibility layer.
+      #
+      # For v0.14 API, use `driver.logs` instead.
+      #
       def capture_log(driver)
         tmp = driver.instance.log.out
         driver.instance.log.out = StringIO.new
@@ -81,6 +120,15 @@ EOT
       ensure
         driver.instance.log.out = tmp
       end
+
+      def capture_stdout
+        out = StringIO.new
+        $stdout = out
+        yield
+        out.string.force_encoding('utf-8')
+      ensure
+        $stdout = STDOUT
+      end
     end
   end
 end

data/lib/fluent/test/input_test.rb

@@ -14,29 +14,22 @@
 #    limitations under the License.
 #
 
-require 'fluent/input'
-require 'fluent/buffer'
 require 'fluent/engine'
 require 'fluent/time'
 require 'fluent/test/base'
 
 module Fluent
-  class FileBuffer < BasicBuffer
-    def self.clear_buffer_paths
-      @@buffer_paths = {}
-    end
-  end
-
   module Test
     class InputTestDriver < TestDriver
       def initialize(klass, &block)
-        FileBuffer.clear_buffer_paths
         super(klass, &block)
         @emit_streams = []
+        @event_streams = []
         @expects = nil
         # for checking only the number of emitted records during run
         @expected_emits_length = nil
         @run_timeout = 5
+        @run_post_conditions = []
       end
 
       def expect_emit(tag, time, record)
@@ -50,7 +43,7 @@ module Fluent
 
       attr_accessor :expected_emits_length
       attr_accessor :run_timeout
-      attr_reader :emit_streams
+      attr_reader :emit_streams, :event_streams
 
       def emits
         all = []
@@ -82,7 +75,6 @@ module Fluent
 
       def register_run_post_condition(&block)
         if block
-          @run_post_conditions ||= []
           @run_post_conditions << block
         end
       end
@@ -108,14 +100,26 @@ module Fluent
         false
       end
 
+      module EmitStreamWrapper
+        def emit_stream_callee=(method)
+          @emit_stream_callee = method
+        end
+        def emit_stream(tag, es)
+          @emit_stream_callee.call(tag, es)
+        end
+      end
+
       def run(num_waits = 10, &block)
         m = method(:emit_stream)
-        Engine.define_singleton_method(:emit_stream) {|tag,es|
-          m.call(tag, es)
-        }
-        instance.router.define_singleton_method(:emit_stream) {|tag,es|
-          m.call(tag, es)
-        }
+        unless Engine.singleton_class.ancestors.include?(EmitStreamWrapper)
+          Engine.singleton_class.prepend EmitStreamWrapper
+        end
+        Engine.emit_stream_callee = m
+        unless instance.router.singleton_class.ancestors.include?(EmitStreamWrapper)
+          instance.router.singleton_class.prepend EmitStreamWrapper
+        end
+        instance.router.emit_stream_callee = m
+
         super(num_waits) {
           block.call if block
 
@@ -132,7 +136,7 @@ module Fluent
               end
             end
 
-            # Set runnning timeout to avoid infinite loop caused by some errors.
+            # Set running timeout to avoid infinite loop caused by some errors.
             started_at = Time.now
             register_run_breaking_condition do
               Time.now >= started_at + @run_timeout
@@ -146,7 +150,10 @@ module Fluent
 
               tag, events = @emit_streams[j]
               events.each do |time, record|
-                assert_equal(@expects[i], [tag, time, record]) if @expects
+                if @expects
+                  assert_equal(@expects[i], [tag, time, record])
+                  assert_equal_event_time(@expects[i][1], time) if @expects[i][1].is_a?(Fluent::EventTime)
+                end
                 i += 1
               end
               j += 1
@@ -159,6 +166,7 @@ module Fluent
 
       private
       def emit_stream(tag, es)
+        @event_streams << es
         @emit_streams << [tag, es.to_a]
       end
     end

data/lib/fluent/test/log.rb

@@ -0,0 +1,79 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'serverengine'
+require 'fluent/log'
+
+module Fluent
+  module Test
+    class DummyLogDevice
+      attr_reader :logs
+      attr_accessor :flush_logs
+
+      def initialize
+        @logs = []
+        @flush_logs = true
+        @use_stderr = false
+      end
+
+      def reset
+        @logs = [] if @flush_logs
+      end
+
+      def tty?
+        false
+      end
+
+      def puts(*args)
+        args.each{ |arg| write(arg + "\n") }
+      end
+
+      def write(message)
+        if @use_stderr
+          STDERR.write message
+        end
+        @logs.push message
+      end
+
+      def flush
+        true
+      end
+
+      def close
+        true
+      end
+    end
+
+    class TestLogger < Fluent::PluginLogger
+      def initialize
+        @logdev = DummyLogDevice.new
+        dl_opts = {}
+        dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO
+        logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
+        log = Fluent::Log.new(logger)
+        super(log)
+      end
+
+      def reset
+        @logdev.reset
+      end
+
+      def logs
+        @logdev.logs
+      end
+    end
+  end
+end

data/lib/fluent/test/output_test.rb

@@ -41,11 +41,9 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Time.now)
-        es = OneEventStream.new(time.to_i, record)
-        chain = TestOutputChain.new
-        @instance.emit(@tag, es, chain)
-        assert_equal 1, chain.called
+      def emit(record, time=Engine.now)
+        es = OneEventStream.new(time, record)
+        @instance.emit_events(@tag, es)
       end
     end
 
@@ -64,8 +62,8 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Time.now)
-        @entries << [time.to_i, record]
+      def emit(record, time=Engine.now)
+        @entries << [time, record]
         self
       end
 
@@ -85,18 +83,13 @@ module Fluent
             assert_equal(@expected_buffer, buffer)
           end
 
-          case
-          when @instance.is_a?(Fluent::ObjectBufferedOutput)
-            key = @tag
-          when @instance.respond_to?(:time_slicer)
-            # this block is only for test_out_file
-            time, _record = @entries.first
-            key = @instance.time_slicer.call(time)
-          else
-            key = ''
-          end
-          chunk = @instance.buffer.new_chunk(key)
-          chunk << buffer
+          chunk = if @instance.instance_eval{ @chunk_key_tag }
+                    @instance.buffer.generate_chunk(@instance.metadata(@tag, nil, nil)).staged!
+                  else
+                    @instance.buffer.generate_chunk(@instance.metadata(nil, nil, nil)).staged!
+                  end
+          chunk.concat(buffer, es.size)
+
           begin
             result = @instance.write(chunk)
           ensure
@@ -110,18 +103,15 @@ module Fluent
     class TimeSlicedOutputTestDriver < InputTestDriver
       def initialize(klass, tag='test', &block)
         super(klass, &block)
-        @entries = {}
+        @entries = []
         @expected_buffer = nil
         @tag = tag
       end
 
       attr_accessor :tag
 
-      def emit(record, time=Time.now)
-        slicer = @instance.instance_eval{@time_slicer}
-        key = slicer.call(time.to_i)
-        @entries[key] = [] unless @entries.has_key?(key)
-        @entries[key] << [time.to_i, record]
+      def emit(record, time=Engine.now)
+        @entries << [time, record]
         self
       end
 
@@ -135,30 +125,29 @@ module Fluent
           block.call if block
 
           buffer = ''
-          @entries.keys.each {|key|
-            es = ArrayEventStream.new(@entries[key])
-            @instance.emit(@tag, es, NullOutputChain.instance)
-            buffer << @instance.format_stream(@tag, es)
-          }
+          lines = {}
+          # v0.12 TimeSlicedOutput doesn't call #format_stream
+          @entries.each do |time, record|
+            meta = @instance.metadata(@tag, time, record)
+            line = @instance.format(@tag, time, record)
+            buffer << line
+            lines[meta] ||= []
+            lines[meta] << line
+          end
 
           if @expected_buffer
             assert_equal(@expected_buffer, buffer)
           end
 
-          chunks = []
-          @instance.instance_eval do
-            @buffer.instance_eval{ @map.keys }.each do |key|
-              @buffer.push(key)
-              chunks << @buffer.instance_eval{ @queue.pop }
-            end
-          end
-          chunks.each { |chunk|
+          lines.keys.each do |meta|
+            chunk = @instance.buffer.generate_chunk(meta).staged!
+            chunk.append(lines[meta])
             begin
               result.push(@instance.write(chunk))
             ensure
               chunk.purge
             end
-          }
+          end
         }
         result
       end

data/lib/fluent/test/parser_test.rb

@@ -25,7 +25,9 @@ module Fluent
           if block
             # Create new class for test w/ overwritten methods
             #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass_or_str.name
             klass_or_str = Class.new(klass_or_str)
+            klass_or_str.define_singleton_method("name") { klass_name }
             klass_or_str.module_eval(&block)
           end
           case klass_or_str.instance_method(:initialize).arity
@@ -36,7 +38,7 @@ module Fluent
             @instance = klass_or_str.new(format, conf)
           end
         elsif klass_or_str.is_a?(String)
-          @instance = TextParser::TEMPLATE_REGISTRY.lookup(klass_or_str).call
+          @instance = Fluent::Plugin.new_parser(klass_or_str)
         else
           @instance = klass_or_str
         end

data/lib/fluent/test/startup_shutdown.rb

@@ -0,0 +1,46 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'serverengine'
+require 'fileutils'
+
+module Fluent
+  module Test
+    module StartupShutdown
+      def startup
+        socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      end
+
+      def shutdown
+        @server.close
+      end
+
+      def self.setup
+        @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        @server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+      end
+
+      def self.teardown
+        @server.close
+        # on Windows, socket_manager_path is a TCP port number
+        FileUtils.rm_f @socket_manager_path unless Fluent.windows?
+      end
+    end
+  end
+end

data/lib/fluent/test.rb

@@ -16,11 +16,43 @@
 
 require 'test/unit'
 require 'fluent/env' # for Fluent.windows?
+require 'fluent/test/log'
 require 'fluent/test/base'
 require 'fluent/test/input_test'
 require 'fluent/test/output_test'
 require 'fluent/test/filter_test'
 require 'fluent/test/parser_test'
 require 'fluent/test/formatter_test'
+require 'serverengine'
 
-$log ||= Fluent::Log.new(Fluent::Test::DummyLogDevice.new)
+
+module Fluent
+  module Test
+    def self.dummy_logger
+      dl_opts = {log_level: ServerEngine::DaemonLogger::INFO}
+      logdev = Fluent::Test::DummyLogDevice.new
+      logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+      Fluent::Log.new(logger)
+    end
+
+    def self.setup
+      ENV['SERVERENGINE_WORKER_ID'] = '0'
+
+      $log = dummy_logger
+
+      Fluent.__send__(:remove_const, :Engine)
+      engine = Fluent.const_set(:Engine, EngineClass.new).init(SystemConfig.new)
+
+      engine.define_singleton_method(:now=) {|n|
+        @now = n
+      }
+      engine.define_singleton_method(:now) {
+        @now ||= super()
+      }
+
+      nil
+    end
+  end
+end
+
+$log ||= Fluent::Test.dummy_logger