fluentd 0.12.43 → 0.14.0

data/lib/fluent/test/driver/filter.rb

@@ -0,0 +1,35 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Test
+    module Driver
+      class Filter < Base
+        include EventFeeder
+
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Filter" unless @instance.is_a? Fluent::Plugin::Filter
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/input.rb

@@ -0,0 +1,31 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base'
+require 'fluent/plugin/input'
+
+module Fluent
+  module Test
+    module Driver
+      class Input < Base
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Input" unless @instance.is_a? Fluent::Plugin::Input
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/output.rb

@@ -0,0 +1,78 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/plugin/output'
+
+module Fluent
+  module Test
+    module Driver
+      class Output < Base
+        include EventFeeder
+
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Output" unless @instance.is_a? Fluent::Plugin::Output
+          @instance.in_tests = true
+          @flush_buffer_at_cleanup = nil
+          @format_hook = nil
+          @format_results = []
+        end
+
+        def run(flush: true, **kwargs, &block)
+          @flush_buffer_at_cleanup = flush
+          super(**kwargs, &block)
+        end
+
+        def run_actual(**kwargs, &block)
+          super(**kwargs, &block)
+          if @flush_buffer_at_cleanup
+            @instance.force_flush
+          end
+        end
+
+        def formatted
+          @format_results
+        end
+
+        def flush
+          @instance.force_flush
+        end
+
+        def instance_hook_after_started
+          super
+
+          # it's decided after #start whether output plugin instances use @custom_format or not.
+          if @instance.instance_eval{ @custom_format }
+            @format_hook = format_hook = ->(result){ @format_results << result }
+            m = Module.new do
+              define_method(:format) do |tag, time, record|
+                result = super(tag, time, record)
+                format_hook.call(result)
+                result
+              end
+            end
+            @instance.singleton_class.module_eval do
+              prepend m
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/test_event_router.rb

@@ -0,0 +1,45 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+
+module Fluent
+  module Test
+    module Driver
+      class TestEventRouter
+        def initialize(driver)
+          @driver = driver
+        end
+
+        def emit(tag, time, record)
+          @driver.emit_event_stream(tag, OneEventStream.new(time, record))
+        end
+
+        def emit_array(tag, array)
+          @driver.emit_event_stream(tag, ArrayEventStream.new(array))
+        end
+
+        def emit_stream(tag, es)
+          @driver.emit_event_stream(tag, es)
+        end
+
+        def emit_error_event(tag, time, record, error)
+          @driver.emit_error_event(tag, time, record, error)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/filter_test.rb

@@ -16,7 +16,6 @@
 
 require 'fluent/test/base'
 require 'fluent/event'
-require 'fluent/filter'
 
 module Fluent
   module Test

data/lib/fluent/test/formatter_test.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/formatter'
 require 'fluent/config'
+require 'fluent/plugin'
 
 module Fluent
   module Test
@@ -30,7 +31,7 @@ module Fluent
           end
           @instance = klass_or_str.new
         elsif klass_or_str.is_a?(String)
-          @instance = TextFormatter::TEMPLATE_REGISTRY.lookup(klass_or_str).call
+          @instance = Fluent::Plugin.new_formatter(klass_or_str)
         else
           @instance = klass_or_str
         end

data/lib/fluent/test/input_test.rb

@@ -14,29 +14,21 @@
 #    limitations under the License.
 #
 
-require 'fluent/input'
-require 'fluent/buffer'
 require 'fluent/engine'
 require 'fluent/time'
 require 'fluent/test/base'
 
 module Fluent
-  class FileBuffer < BasicBuffer
-    def self.clear_buffer_paths
-      @@buffer_paths = {}
-    end
-  end
-
   module Test
     class InputTestDriver < TestDriver
       def initialize(klass, &block)
-        FileBuffer.clear_buffer_paths
         super(klass, &block)
         @emit_streams = []
         @expects = nil
         # for checking only the number of emitted records during run
         @expected_emits_length = nil
         @run_timeout = 5
+        @run_post_conditions = []
       end
 
       def expect_emit(tag, time, record)
@@ -82,7 +74,6 @@ module Fluent
 
       def register_run_post_condition(&block)
         if block
-          @run_post_conditions ||= []
           @run_post_conditions << block
         end
       end
@@ -108,14 +99,26 @@ module Fluent
         false
       end
 
+      module EmitStreamWrapper
+        def emit_stream_callee=(method)
+          @emit_stream_callee = method
+        end
+        def emit_stream(tag, es)
+          @emit_stream_callee.call(tag, es)
+        end
+      end
+
       def run(num_waits = 10, &block)
         m = method(:emit_stream)
-        Engine.define_singleton_method(:emit_stream) {|tag,es|
-          m.call(tag, es)
-        }
-        instance.router.define_singleton_method(:emit_stream) {|tag,es|
-          m.call(tag, es)
-        }
+        (class << Engine; self; end).module_eval do
+          prepend EmitStreamWrapper
+        end
+        Engine.emit_stream_callee = m
+        (class << instance.router; self; end).module_eval do
+          prepend EmitStreamWrapper
+        end
+        instance.router.emit_stream_callee = m
+
         super(num_waits) {
           block.call if block
 
@@ -146,7 +149,10 @@ module Fluent
 
               tag, events = @emit_streams[j]
               events.each do |time, record|
-                assert_equal(@expects[i], [tag, time, record]) if @expects
+                if @expects
+                  assert_equal(@expects[i], [tag, time, record])
+                  assert_equal_event_time(@expects[i][1], time) if @expects[i][1].is_a?(Fluent::EventTime)
+                end
                 i += 1
               end
               j += 1

data/lib/fluent/test/output_test.rb

@@ -41,11 +41,9 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Time.now)
-        es = OneEventStream.new(time.to_i, record)
-        chain = TestOutputChain.new
-        @instance.emit(@tag, es, chain)
-        assert_equal 1, chain.called
+      def emit(record, time=Engine.now)
+        es = OneEventStream.new(time, record)
+        @instance.emit_events(@tag, es)
       end
     end
 
@@ -64,8 +62,8 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Time.now)
-        @entries << [time.to_i, record]
+      def emit(record, time=Engine.now)
+        @entries << [time, record]
         self
       end
 
@@ -85,18 +83,13 @@ module Fluent
             assert_equal(@expected_buffer, buffer)
           end
 
-          case
-          when @instance.is_a?(Fluent::ObjectBufferedOutput)
-            key = @tag
-          when @instance.respond_to?(:time_slicer)
-            # this block is only for test_out_file
-            time, _record = @entries.first
-            key = @instance.time_slicer.call(time)
-          else
-            key = ''
-          end
-          chunk = @instance.buffer.new_chunk(key)
-          chunk << buffer
+          chunk = if @instance.instance_eval{ @chunk_key_tag }
+                    @instance.buffer.generate_chunk(@instance.metadata(@tag, nil, nil))
+                  else
+                    @instance.buffer.generate_chunk(@instance.metadata(nil, nil, nil))
+                  end
+          chunk.concat(buffer, es.size)
+
           begin
             result = @instance.write(chunk)
           ensure
@@ -110,18 +103,15 @@ module Fluent
     class TimeSlicedOutputTestDriver < InputTestDriver
       def initialize(klass, tag='test', &block)
         super(klass, &block)
-        @entries = {}
+        @entries = []
         @expected_buffer = nil
         @tag = tag
       end
 
       attr_accessor :tag
 
-      def emit(record, time=Time.now)
-        slicer = @instance.instance_eval{@time_slicer}
-        key = slicer.call(time.to_i)
-        @entries[key] = [] unless @entries.has_key?(key)
-        @entries[key] << [time.to_i, record]
+      def emit(record, time=Engine.now)
+        @entries << [time, record]
         self
       end
 
@@ -135,30 +125,29 @@ module Fluent
           block.call if block
 
           buffer = ''
-          @entries.keys.each {|key|
-            es = ArrayEventStream.new(@entries[key])
-            @instance.emit(@tag, es, NullOutputChain.instance)
-            buffer << @instance.format_stream(@tag, es)
-          }
+          lines = {}
+          # v0.12 TimeSlicedOutput doesn't call #format_stream
+          @entries.each do |time, record|
+            meta = @instance.metadata(@tag, time, record)
+            line = @instance.format(@tag, time, record)
+            buffer << line
+            lines[meta] ||= []
+            lines[meta] << line
+          end
 
           if @expected_buffer
             assert_equal(@expected_buffer, buffer)
           end
 
-          chunks = []
-          @instance.instance_eval do
-            @buffer.instance_eval{ @map.keys }.each do |key|
-              @buffer.push(key)
-              chunks << @buffer.instance_eval{ @queue.pop }
-            end
-          end
-          chunks.each { |chunk|
+          lines.keys.each do |meta|
+            chunk = @instance.buffer.generate_chunk(meta)
+            chunk.append(lines[meta])
             begin
               result.push(@instance.write(chunk))
             ensure
               chunk.purge
             end
-          }
+          end
         }
         result
       end

data/lib/fluent/test/parser_test.rb

@@ -36,7 +36,7 @@ module Fluent
             @instance = klass_or_str.new(format, conf)
           end
         elsif klass_or_str.is_a?(String)
-          @instance = TextParser::TEMPLATE_REGISTRY.lookup(klass_or_str).call
+          @instance = Fluent::Plugin.new_parser(klass_or_str)
         else
           @instance = klass_or_str
         end

data/lib/fluent/time.rb

@@ -1 +1,104 @@
-# placeholder for v0.14 time module
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'time'
+require 'msgpack'
+
+module Fluent
+  class EventTime
+    TYPE = 0
+
+    def initialize(sec, nsec = 0)
+      @sec = sec
+      @nsec = nsec
+    end
+
+    def ==(other)
+      if other.is_a?(Fluent::EventTime)
+        @sec == other.sec
+      else
+        @sec == other
+      end
+    end
+
+    def sec
+      @sec
+    end
+
+    def nsec
+      @nsec
+    end
+
+    def to_int
+      @sec
+    end
+
+    # for Time.at
+    def to_r
+      Rational(@sec * 1_000_000_000 + @nsec, 1_000_000_000)
+    end
+
+    # for > and others
+    def coerce(other)
+      [other, @sec]
+    end
+
+    def to_s
+      @sec.to_s
+    end
+
+    def to_json(*args)
+      @sec.to_s
+    end
+
+    def to_msgpack(io = nil)
+      @sec.to_msgpack(io)
+    end
+
+    def to_msgpack_ext
+      [@sec, @nsec].pack('NN')
+    end
+
+    def self.from_msgpack_ext(data)
+      new(*data.unpack('NN'))
+    end
+
+    def self.from_time(time)
+      Fluent::EventTime.new(time.to_i, time.nsec)
+    end
+
+    def self.eq?(a, b)
+      if a.is_a?(Fluent::EventTime) && b.is_a?(Fluent::EventTime)
+        a.sec == b.sec && a.nsec == b.nsec
+      else
+        a == b
+      end
+    end
+
+    def self.now
+      from_time(Time.now)
+    end
+
+    def self.parse(*args)
+      from_time(Time.parse(*args))
+    end
+
+    ## TODO: For performance, implement +, -, and so on
+    def method_missing(name, *args, &block)
+      @sec.send(name, *args, &block)
+    end
+  end
+end