dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/test/driver/event_feeder.rb

@@ -0,0 +1,98 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+require 'fluent/time'
+
+module Fluent
+  module Test
+    module Driver
+      module EventFeeder
+        def initialize(klass, opts: {}, &block)
+          super
+          @default_tag = nil
+          @feed_method = nil
+        end
+
+        def run(default_tag: nil, **kwargs, &block)
+          @feed_method = if @instance.respond_to?(:filter_stream)
+                           :filter_stream
+                         else
+                           :emit_events
+                         end
+          if default_tag
+            @default_tag = default_tag
+          end
+          super(**kwargs, &block)
+        end
+
+        def feed_to_plugin(tag, es)
+          @instance.__send__(@feed_method, tag, es)
+        end
+
+        # d.run do
+        #   d.feed('tag', time, {record})
+        #   d.feed('tag', [ [time, {record}], [time, {record}], ... ])
+        #   d.feed('tag', es)
+        # end
+        # d.run(default_tag: 'tag') do
+        #   d.feed({record})
+        #   d.feed(time, {record})
+        #   d.feed([ [time, {record}], [time, {record}], ... ])
+        #   d.feed(es)
+        # end
+        def feed(*args)
+          case args.size
+          when 1
+            raise ArgumentError, "tag not specified without default_tag" unless @default_tag
+            case args.first
+            when Fluent::EventStream
+              feed_to_plugin(@default_tag, args.first)
+            when Array
+              feed_to_plugin(@default_tag, ArrayEventStream.new(args.first))
+            when Hash
+              record = args.first
+              time = Fluent::EventTime.now
+              feed_to_plugin(@default_tag, OneEventStream.new(time, record))
+            else
+              raise ArgumentError, "unexpected events object (neither event(Hash), EventStream nor Array): #{args.first.class}"
+            end
+          when 2
+            if args[0].is_a?(String) && (args[1].is_a?(Array) || args[1].is_a?(Fluent::EventStream))
+              tag, es = args
+              es = ArrayEventStream.new(es) if es.is_a?(Array)
+              feed_to_plugin(tag, es)
+            elsif @default_tag && (args[0].is_a?(Fluent::EventTime) || args[0].is_a?(Integer)) && args[1].is_a?(Hash)
+              time, record = args
+              feed_to_plugin(@default_tag, OneEventStream.new(time, record))
+            else
+              raise ArgumentError, "unexpected values of argument: #{args[0].class}, #{args[1].class}"
+            end
+          when 3
+            tag, time, record = args
+            if tag.is_a?(String) && (time.is_a?(Fluent::EventTime) || time.is_a?(Integer)) && record.is_a?(Hash)
+              feed_to_plugin(tag, OneEventStream.new(time, record))
+            else
+              raise ArgumentError, "unexpected values of argument: #{tag.class}, #{time.class}, #{record.class}"
+            end
+          else
+            raise ArgumentError, "unexpected number of arguments: #{args}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/filter.rb

@@ -0,0 +1,57 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owner'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Test
+    module Driver
+      class Filter < BaseOwner
+        include EventFeeder
+
+        attr_reader :filtered
+
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Filter" unless @instance.is_a? Fluent::Plugin::Filter
+          @filtered = []
+        end
+
+        def filtered_records
+          @filtered.map {|_time, record| record }
+        end
+
+        def instance_hook_after_started
+          super
+          filter_hook = ->(time, record) { @filtered << [time, record] }
+          m = Module.new do
+            define_method(:filter_stream) do |tag, es|
+              new_es = super(tag, es)
+              new_es.each do |time, record|
+                filter_hook.call(time, record)
+              end
+              new_es
+            end
+          end
+          @instance.singleton_class.prepend(m)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/formatter.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owned'
+
+module Fluent
+  module Test
+    module Driver
+      class Formatter < BaseOwned
+        def initialize(klass, **kwargs, &block)
+          super
+          @section_name = "format"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/input.rb

@@ -0,0 +1,31 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owner'
+require 'fluent/plugin/input'
+
+module Fluent
+  module Test
+    module Driver
+      class Input < BaseOwner
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Input" unless @instance.is_a? Fluent::Plugin::Input
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/multi_output.rb

@@ -0,0 +1,53 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owner'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/plugin/multi_output'
+
+module Fluent
+  module Test
+    module Driver
+      class MultiOutput < BaseOwner
+        include EventFeeder
+
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::MultiOutput" unless @instance.is_a? Fluent::Plugin::MultiOutput
+          @flush_buffer_at_cleanup = nil
+        end
+
+        def run(flush: true, **kwargs, &block)
+          @flush_buffer_at_cleanup = flush
+          super(**kwargs, &block)
+        end
+
+        def run_actual(**kwargs, &block)
+          val = super(**kwargs, &block)
+          if @flush_buffer_at_cleanup
+            @instance.outputs.each{|o| o.force_flush }
+          end
+          val
+        end
+
+        def flush
+          @instance.outputs.each{|o| o.force_flush }
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/output.rb

@@ -0,0 +1,102 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owner'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/plugin/output'
+require 'timeout'
+
+module Fluent
+  module Test
+    module Driver
+      class Output < BaseOwner
+        include EventFeeder
+
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Output" unless @instance.is_a? Fluent::Plugin::Output
+          @flush_buffer_at_cleanup = nil
+          @wait_flush_completion = nil
+          @force_flush_retry = nil
+          @format_hook = nil
+          @format_results = []
+        end
+
+        def run(flush: true, wait_flush_completion: true, force_flush_retry: false, **kwargs, &block)
+          @flush_buffer_at_cleanup = flush
+          @wait_flush_completion = wait_flush_completion
+          @force_flush_retry = force_flush_retry
+          super(**kwargs, &block)
+        end
+
+        def run_actual(**kwargs, &block)
+          if @force_flush_retry
+            @instance.retry_for_error_chunk = true
+          end
+          val = super(**kwargs, &block)
+          if @flush_buffer_at_cleanup
+            self.flush
+          end
+          val
+        ensure
+          @instance.retry_for_error_chunk = false
+        end
+
+        def formatted
+          @format_results
+        end
+
+        def flush
+          @instance.force_flush
+          wait_flush_completion if @wait_flush_completion
+        end
+
+        def wait_flush_completion
+          buffer_queue = ->(){ @instance.buffer && @instance.buffer.queue.size > 0 }
+          dequeued_chunks = ->(){
+            @instance.dequeued_chunks_mutex &&
+            @instance.dequeued_chunks &&
+            @instance.dequeued_chunks_mutex.synchronize{ @instance.dequeued_chunks.size > 0 }
+          }
+
+          Timeout.timeout(10) do
+            while buffer_queue.call || dequeued_chunks.call
+              sleep 0.1
+            end
+          end
+        end
+
+        def instance_hook_after_started
+          super
+
+          # it's decided after #start whether output plugin instances use @custom_format or not.
+          if @instance.instance_eval{ @custom_format }
+            @format_hook = format_hook = ->(result){ @format_results << result }
+            m = Module.new do
+              define_method(:format) do |tag, time, record|
+                result = super(tag, time, record)
+                format_hook.call(result)
+                result
+              end
+            end
+            @instance.singleton_class.prepend m
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/parser.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owned'
+
+module Fluent
+  module Test
+    module Driver
+      class Parser < BaseOwned
+        def initialize(klass, **kwargs, &block)
+          super
+          @section_name = "parse"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/storage.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owned'
+
+module Fluent
+  module Test
+    module Driver
+      class Storage < BaseOwned
+        def initialize(klass, **kwargs, &block)
+          super
+          @section_name = "storage"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/test_event_router.rb

@@ -0,0 +1,45 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+
+module Fluent
+  module Test
+    module Driver
+      class TestEventRouter
+        def initialize(driver)
+          @driver = driver
+        end
+
+        def emit(tag, time, record)
+          @driver.emit_event_stream(tag, OneEventStream.new(time, record))
+        end
+
+        def emit_array(tag, array)
+          @driver.emit_event_stream(tag, ArrayEventStream.new(array))
+        end
+
+        def emit_stream(tag, es)
+          @driver.emit_event_stream(tag, es)
+        end
+
+        def emit_error_event(tag, time, record, error)
+          @driver.emit_error_event(tag, time, record, error)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/filter_test.rb

@@ -0,0 +1,77 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/base'
+require 'fluent/event'
+
+module Fluent
+  module Test
+    class FilterTestDriver < TestDriver
+      def initialize(klass, tag = 'filter.test', &block)
+        super(klass, &block)
+        @tag = tag
+        @events = {}
+        @filtered = MultiEventStream.new
+      end
+
+      attr_reader :filtered
+      attr_accessor :tag
+
+      def emit(record, time = EventTime.now)
+        emit_with_tag(@tag, record, time)
+      end
+      alias_method :filter, :emit
+
+      def emit_with_tag(tag, record, time = EventTime.now)
+        @events[tag] ||= MultiEventStream.new
+        @events[tag].add(time, record)
+      end
+      alias_method :filter_with_tag, :emit_with_tag
+
+      def filter_stream(es)
+        filter_stream_with_tag(@tag, es)
+      end
+
+      def filter_stream_with_tag(tag, es)
+        @events[tag] = es
+      end
+
+      def filtered_as_array
+        all = []
+        @filtered.each { |time, record|
+          all << [@tag, time, record]
+        }
+        all
+      end
+      alias_method :emits, :filtered_as_array # emits is for consistent with other drivers
+
+      # Almost filters don't use threads so default is 0. It reduces test time.
+      def run(num_waits = 0, &block)
+        super(num_waits) {
+          block.call if block
+
+          @events.each { |tag, es|
+            processed = @instance.filter_stream(tag, es)
+            processed.each { |time, record|
+              @filtered.add(time, record)
+            }
+          }
+        }
+        self
+      end
+    end
+  end
+end

data/lib/fluent/test/formatter_test.rb

@@ -0,0 +1,65 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/formatter'
+require 'fluent/config'
+require 'fluent/plugin'
+
+module Fluent
+  module Test
+    class FormatterTestDriver
+      def initialize(klass_or_str, proc=nil, &block)
+        if klass_or_str.is_a?(Class)
+          if block
+            # Create new class for test w/ overwritten methods
+            #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass_or_str.name
+            klass_or_str = Class.new(klass_or_str)
+            klass_or_str.define_singleton_method("name") { klass_name }
+            klass_or_str.module_eval(&block)
+          end
+          @instance = klass_or_str.new
+        elsif klass_or_str.is_a?(String)
+          @instance = Fluent::Plugin.new_formatter(klass_or_str)
+        else
+          @instance = klass_or_str
+        end
+        @config = Config.new
+      end
+
+      attr_reader :instance, :config
+
+      def configure(conf)
+        case conf
+        when Fluent::Config::Element
+          @config = conf
+        when String
+          @config = Config.parse(conf, 'fluent.conf')
+        when Hash
+          @config = Config::Element.new('ROOT', '', conf, [])
+        else
+          raise "Unknown type... #{conf}"
+        end
+        @instance.configure(@config)
+        self
+      end
+
+      def format(tag, time, record)
+        @instance.format(tag, time, record)
+      end
+    end
+  end
+end