fluentd 0.12.40 → 1.6.2

data/lib/fluent/test/driver/base_owner.rb

@@ -0,0 +1,135 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base'
+require 'fluent/test/driver/test_event_router'
+
+module Fluent
+  module Test
+    module Driver
+      class BaseOwner < Base
+        def initialize(klass, opts: {}, &block)
+          super
+
+          if opts
+            @instance.system_config_override(opts)
+          end
+          @instance.log = TestLogger.new
+          @logs = @instance.log.out.logs
+
+          @event_streams = nil
+          @error_events = nil
+        end
+
+        def configure(conf, syntax: :v1)
+          if conf.is_a?(Fluent::Config::Element)
+            @config = conf
+          else
+            @config = Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
+          end
+
+          if @instance.respond_to?(:router=)
+            @event_streams = []
+            @error_events = []
+
+            driver = self
+            mojule = Module.new do
+              define_method(:event_emitter_router) do |label_name|
+                TestEventRouter.new(driver)
+              end
+            end
+            @instance.singleton_class.prepend mojule
+          end
+
+          @instance.configure(@config)
+          self
+        end
+
+        Emit = Struct.new(:tag, :es)
+        ErrorEvent = Struct.new(:tag, :time, :record, :error)
+
+        # via TestEventRouter
+        def emit_event_stream(tag, es)
+          @event_streams << Emit.new(tag, es)
+        end
+
+        def emit_error_event(tag, time, record, error)
+          @error_events << ErrorEvent.new(tag, time, record, error)
+        end
+
+        def events(tag: nil)
+          if block_given?
+            event_streams(tag: tag) do |t, es|
+              es.each do |time, record|
+                yield t, time, record
+              end
+            end
+          else
+            list = []
+            event_streams(tag: tag) do |t, es|
+              es.each do |time, record|
+                list << [t, time, record]
+              end
+            end
+            list
+          end
+        end
+
+        def event_streams(tag: nil)
+          return [] if @event_streams.nil?
+          selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }
+          if block_given?
+            selected.each do |e|
+              yield e.tag, e.es
+            end
+          else
+            selected.map{|e| [e.tag, e.es] }
+          end
+        end
+
+        def emit_count
+          @event_streams.size
+        end
+
+        def record_count
+          @event_streams.reduce(0) {|a, e| a + e.es.size }
+        end
+
+        def error_events(tag: nil)
+          selected = @error_events.select{|e| tag.nil? ? true : e.tag == tag }
+          if block_given?
+            selected.each do |e|
+              yield e.tag, e.time, e.record, e.error
+            end
+          else
+            selected.map{|e| [e.tag, e.time, e.record, e.error] }
+          end
+        end
+
+        def run(expect_emits: nil, expect_records: nil, timeout: nil, start: true, shutdown: true, &block)
+          if expect_emits
+            @run_post_conditions << ->(){ emit_count >= expect_emits }
+          end
+          if expect_records
+            @run_post_conditions << ->(){ record_count >= expect_records }
+          end
+
+          super(timeout: timeout, start: start, shutdown: shutdown, &block)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/event_feeder.rb

@@ -0,0 +1,98 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+require 'fluent/time'
+
+module Fluent
+  module Test
+    module Driver
+      module EventFeeder
+        def initialize(klass, opts: {}, &block)
+          super
+          @default_tag = nil
+          @feed_method = nil
+        end
+
+        def run(default_tag: nil, **kwargs, &block)
+          @feed_method = if @instance.respond_to?(:filter_stream)
+                           :filter_stream
+                         else
+                           :emit_events
+                         end
+          if default_tag
+            @default_tag = default_tag
+          end
+          super(**kwargs, &block)
+        end
+
+        def feed_to_plugin(tag, es)
+          @instance.__send__(@feed_method, tag, es)
+        end
+
+        # d.run do
+        #   d.feed('tag', time, {record})
+        #   d.feed('tag', [ [time, {record}], [time, {record}], ... ])
+        #   d.feed('tag', es)
+        # end
+        # d.run(default_tag: 'tag') do
+        #   d.feed({record})
+        #   d.feed(time, {record})
+        #   d.feed([ [time, {record}], [time, {record}], ... ])
+        #   d.feed(es)
+        # end
+        def feed(*args)
+          case args.size
+          when 1
+            raise ArgumentError, "tag not specified without default_tag" unless @default_tag
+            case args.first
+            when Fluent::EventStream
+              feed_to_plugin(@default_tag, args.first)
+            when Array
+              feed_to_plugin(@default_tag, ArrayEventStream.new(args.first))
+            when Hash
+              record = args.first
+              time = Fluent::EventTime.now
+              feed_to_plugin(@default_tag, OneEventStream.new(time, record))
+            else
+              raise ArgumentError, "unexpected events object (neither event(Hash), EventStream nor Array): #{args.first.class}"
+            end
+          when 2
+            if args[0].is_a?(String) && (args[1].is_a?(Array) || args[1].is_a?(Fluent::EventStream))
+              tag, es = args
+              es = ArrayEventStream.new(es) if es.is_a?(Array)
+              feed_to_plugin(tag, es)
+            elsif @default_tag && (args[0].is_a?(Fluent::EventTime) || args[0].is_a?(Integer)) && args[1].is_a?(Hash)
+              time, record = args
+              feed_to_plugin(@default_tag, OneEventStream.new(time, record))
+            else
+              raise ArgumentError, "unexpected values of argument: #{args[0].class}, #{args[1].class}"
+            end
+          when 3
+            tag, time, record = args
+            if tag.is_a?(String) && (time.is_a?(Fluent::EventTime) || time.is_a?(Integer)) && record.is_a?(Hash)
+              feed_to_plugin(tag, OneEventStream.new(time, record))
+            else
+              raise ArgumentError, "unexpected values of argument: #{tag.class}, #{time.class}, #{record.class}"
+            end
+          else
+            raise ArgumentError, "unexpected number of arguments: #{args}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/filter.rb

@@ -0,0 +1,57 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owner'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Test
+    module Driver
+      class Filter < BaseOwner
+        include EventFeeder
+
+        attr_reader :filtered
+
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Filter" unless @instance.is_a? Fluent::Plugin::Filter
+          @filtered = []
+        end
+
+        def filtered_records
+          @filtered.map {|_time, record| record }
+        end
+
+        def instance_hook_after_started
+          super
+          filter_hook = ->(time, record) { @filtered << [time, record] }
+          m = Module.new do
+            define_method(:filter_stream) do |tag, es|
+              new_es = super(tag, es)
+              new_es.each do |time, record|
+                filter_hook.call(time, record)
+              end
+              new_es
+            end
+          end
+          @instance.singleton_class.prepend(m)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/formatter.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owned'
+
+module Fluent
+  module Test
+    module Driver
+      class Formatter < BaseOwned
+        def initialize(klass, **kwargs, &block)
+          super
+          @section_name = "format"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/input.rb

@@ -0,0 +1,31 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owner'
+require 'fluent/plugin/input'
+
+module Fluent
+  module Test
+    module Driver
+      class Input < BaseOwner
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Input" unless @instance.is_a? Fluent::Plugin::Input
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/multi_output.rb

@@ -0,0 +1,53 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owner'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/plugin/multi_output'
+
+module Fluent
+  module Test
+    module Driver
+      class MultiOutput < BaseOwner
+        include EventFeeder
+
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::MultiOutput" unless @instance.is_a? Fluent::Plugin::MultiOutput
+          @flush_buffer_at_cleanup = nil
+        end
+
+        def run(flush: true, **kwargs, &block)
+          @flush_buffer_at_cleanup = flush
+          super(**kwargs, &block)
+        end
+
+        def run_actual(**kwargs, &block)
+          val = super(**kwargs, &block)
+          if @flush_buffer_at_cleanup
+            @instance.outputs.each{|o| o.force_flush }
+          end
+          val
+        end
+
+        def flush
+          @instance.outputs.each{|o| o.force_flush }
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/output.rb

@@ -0,0 +1,102 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owner'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/plugin/output'
+require 'timeout'
+
+module Fluent
+  module Test
+    module Driver
+      class Output < BaseOwner
+        include EventFeeder
+
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Output" unless @instance.is_a? Fluent::Plugin::Output
+          @flush_buffer_at_cleanup = nil
+          @wait_flush_completion = nil
+          @force_flush_retry = nil
+          @format_hook = nil
+          @format_results = []
+        end
+
+        def run(flush: true, wait_flush_completion: true, force_flush_retry: false, **kwargs, &block)
+          @flush_buffer_at_cleanup = flush
+          @wait_flush_completion = wait_flush_completion
+          @force_flush_retry = force_flush_retry
+          super(**kwargs, &block)
+        end
+
+        def run_actual(**kwargs, &block)
+          if @force_flush_retry
+            @instance.retry_for_error_chunk = true
+          end
+          val = super(**kwargs, &block)
+          if @flush_buffer_at_cleanup
+            self.flush
+          end
+          val
+        ensure
+          @instance.retry_for_error_chunk = false
+        end
+
+        def formatted
+          @format_results
+        end
+
+        def flush
+          @instance.force_flush
+          wait_flush_completion if @wait_flush_completion
+        end
+
+        def wait_flush_completion
+          buffer_queue = ->(){ @instance.buffer && @instance.buffer.queue.size > 0 }
+          dequeued_chunks = ->(){
+            @instance.dequeued_chunks_mutex &&
+            @instance.dequeued_chunks &&
+            @instance.dequeued_chunks_mutex.synchronize{ @instance.dequeued_chunks.size > 0 }
+          }
+
+          Timeout.timeout(10) do
+            while buffer_queue.call || dequeued_chunks.call
+              sleep 0.1
+            end
+          end
+        end
+
+        def instance_hook_after_started
+          super
+
+          # it's decided after #start whether output plugin instances use @custom_format or not.
+          if @instance.instance_eval{ @custom_format }
+            @format_hook = format_hook = ->(result){ @format_results << result }
+            m = Module.new do
+              define_method(:format) do |tag, time, record|
+                result = super(tag, time, record)
+                format_hook.call(result)
+                result
+              end
+            end
+            @instance.singleton_class.prepend m
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/parser.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owned'
+
+module Fluent
+  module Test
+    module Driver
+      class Parser < BaseOwned
+        def initialize(klass, **kwargs, &block)
+          super
+          @section_name = "parse"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/test_event_router.rb

@@ -0,0 +1,45 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+
+module Fluent
+  module Test
+    module Driver
+      class TestEventRouter
+        def initialize(driver)
+          @driver = driver
+        end
+
+        def emit(tag, time, record)
+          @driver.emit_event_stream(tag, OneEventStream.new(time, record))
+        end
+
+        def emit_array(tag, array)
+          @driver.emit_event_stream(tag, ArrayEventStream.new(array))
+        end
+
+        def emit_stream(tag, es)
+          @driver.emit_event_stream(tag, es)
+        end
+
+        def emit_error_event(tag, time, record, error)
+          @driver.emit_error_event(tag, time, record, error)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/filter_test.rb

@@ -16,7 +16,6 @@
 
 require 'fluent/test/base'
 require 'fluent/event'
-require 'fluent/filter'
 
 module Fluent
   module Test

data/lib/fluent/test/formatter_test.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/formatter'
 require 'fluent/config'
+require 'fluent/plugin'
 
 module Fluent
   module Test
@@ -25,12 +26,14 @@ module Fluent
           if block
             # Create new class for test w/ overwritten methods
             #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass_or_str.name
             klass_or_str = Class.new(klass_or_str)
+            klass_or_str.define_singleton_method("name") { klass_name }
             klass_or_str.module_eval(&block)
           end
           @instance = klass_or_str.new
         elsif klass_or_str.is_a?(String)
-          @instance = TextFormatter::TEMPLATE_REGISTRY.lookup(klass_or_str).call
+          @instance = Fluent::Plugin.new_formatter(klass_or_str)
         else
           @instance = klass_or_str
         end