fluentd-hubspot 0.14.14.1

data/lib/fluent/test/input_test.rb

@@ -0,0 +1,174 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/engine'
+require 'fluent/time'
+require 'fluent/test/base'
+
+module Fluent
+  module Test
+    class InputTestDriver < TestDriver
+      def initialize(klass, &block)
+        super(klass, &block)
+        @emit_streams = []
+        @event_streams = []
+        @expects = nil
+        # for checking only the number of emitted records during run
+        @expected_emits_length = nil
+        @run_timeout = 5
+        @run_post_conditions = []
+      end
+
+      def expect_emit(tag, time, record)
+        (@expects ||= []) << [tag, time, record]
+        self
+      end
+
+      def expected_emits
+        @expects ||= []
+      end
+
+      attr_accessor :expected_emits_length
+      attr_accessor :run_timeout
+      attr_reader :emit_streams, :event_streams
+
+      def emits
+        all = []
+        @emit_streams.each {|tag,events|
+          events.each {|time,record|
+            all << [tag, time, record]
+          }
+        }
+        all
+      end
+
+      def events
+        all = []
+        @emit_streams.each {|tag,events|
+          all.concat events
+        }
+        all
+      end
+
+      def records
+        all = []
+        @emit_streams.each {|tag,events|
+          events.each {|time,record|
+            all << record
+          }
+        }
+        all
+      end
+
+      def register_run_post_condition(&block)
+        if block
+          @run_post_conditions << block
+        end
+      end
+
+      def register_run_breaking_condition(&block)
+        if block
+          @run_breaking_conditions ||= []
+          @run_breaking_conditions << block
+        end
+      end
+
+      def run_should_stop?
+        # Should stop running if post conditions are not registered.
+        return true unless @run_post_conditions
+
+        # Should stop running if all of the post conditions are true.
+        return true if @run_post_conditions.all? {|proc| proc.call }
+
+        # Should stop running if any of the breaking conditions is true.
+        # In this case, some post conditions may be not true.
+        return true if @run_breaking_conditions && @run_breaking_conditions.any? {|proc| proc.call }
+
+        false
+      end
+
+      module EmitStreamWrapper
+        def emit_stream_callee=(method)
+          @emit_stream_callee = method
+        end
+        def emit_stream(tag, es)
+          @emit_stream_callee.call(tag, es)
+        end
+      end
+
+      def run(num_waits = 10, &block)
+        m = method(:emit_stream)
+        unless Engine.singleton_class.ancestors.include?(EmitStreamWrapper)
+          Engine.singleton_class.prepend EmitStreamWrapper
+        end
+        Engine.emit_stream_callee = m
+        unless instance.router.singleton_class.ancestors.include?(EmitStreamWrapper)
+          instance.router.singleton_class.prepend EmitStreamWrapper
+        end
+        instance.router.emit_stream_callee = m
+
+        super(num_waits) {
+          block.call if block
+
+          if @expected_emits_length || @expects || @run_post_conditions
+            # counters for emits and emit_streams
+            i, j = 0, 0
+
+            # Events of expected length will be emitted at the end.
+            max_length = @expected_emits_length
+            max_length ||= @expects.length if @expects
+            if max_length
+              register_run_post_condition do
+                i == max_length
+              end
+            end
+
+            # Set running timeout to avoid infinite loop caused by some errors.
+            started_at = Time.now
+            register_run_breaking_condition do
+              Time.now >= started_at + @run_timeout
+            end
+
+            until run_should_stop?
+              if j >= @emit_streams.length
+                sleep 0.01
+                next
+              end
+
+              tag, events = @emit_streams[j]
+              events.each do |time, record|
+                if @expects
+                  assert_equal(@expects[i], [tag, time, record])
+                  assert_equal_event_time(@expects[i][1], time) if @expects[i][1].is_a?(Fluent::EventTime)
+                end
+                i += 1
+              end
+              j += 1
+            end
+            assert_equal(@expects.length, i) if @expects
+          end
+        }
+        self
+      end
+
+      private
+      def emit_stream(tag, es)
+        @event_streams << es
+        @emit_streams << [tag, es.to_a]
+      end
+    end
+  end
+end

data/lib/fluent/test/log.rb

@@ -0,0 +1,79 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'serverengine'
+require 'fluent/log'
+
+module Fluent
+  module Test
+    class DummyLogDevice
+      attr_reader :logs
+      attr_accessor :flush_logs
+
+      def initialize
+        @logs = []
+        @flush_logs = true
+        @use_stderr = false
+      end
+
+      def reset
+        @logs = [] if @flush_logs
+      end
+
+      def tty?
+        false
+      end
+
+      def puts(*args)
+        args.each{ |arg| write(arg + "\n") }
+      end
+
+      def write(message)
+        if @use_stderr
+          STDERR.write message
+        end
+        @logs.push message
+      end
+
+      def flush
+        true
+      end
+
+      def close
+        true
+      end
+    end
+
+    class TestLogger < Fluent::PluginLogger
+      def initialize
+        @logdev = DummyLogDevice.new
+        dl_opts = {}
+        dl_opts[:log_level] = ServerEngine::DaemonLogger::INFO
+        logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
+        log = Fluent::Log.new(logger)
+        super(log)
+      end
+
+      def reset
+        @logdev.reset
+      end
+
+      def logs
+        @logdev.logs
+      end
+    end
+  end
+end

data/lib/fluent/test/output_test.rb

@@ -0,0 +1,156 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/engine'
+require 'fluent/event'
+require 'fluent/test/input_test'
+
+module Fluent
+  module Test
+    class TestOutputChain
+      def initialize
+        @called = 0
+      end
+
+      def next
+        @called += 1
+      end
+
+      attr_reader :called
+    end
+
+
+    class OutputTestDriver < InputTestDriver
+      def initialize(klass, tag='test', &block)
+        super(klass, &block)
+        @tag = tag
+      end
+
+      attr_accessor :tag
+
+      def emit(record, time=Engine.now)
+        es = OneEventStream.new(time, record)
+        @instance.emit_events(@tag, es)
+      end
+    end
+
+
+    class BufferedOutputTestDriver < InputTestDriver
+      def initialize(klass, tag='test', &block)
+        super(klass, &block)
+        @entries = []
+        @expected_buffer = nil
+        @tag = tag
+
+        def @instance.buffer
+          @buffer
+        end
+      end
+
+      attr_accessor :tag
+
+      def emit(record, time=Engine.now)
+        @entries << [time, record]
+        self
+      end
+
+      def expect_format(str)
+        (@expected_buffer ||= '') << str
+      end
+
+      def run(num_waits = 10, &block)
+        result = nil
+        super(num_waits) {
+          block.call if block
+
+          es = ArrayEventStream.new(@entries)
+          buffer = @instance.format_stream(@tag, es)
+
+          if @expected_buffer
+            assert_equal(@expected_buffer, buffer)
+          end
+
+          chunk = if @instance.instance_eval{ @chunk_key_tag }
+                    @instance.buffer.generate_chunk(@instance.metadata(@tag, nil, nil)).staged!
+                  else
+                    @instance.buffer.generate_chunk(@instance.metadata(nil, nil, nil)).staged!
+                  end
+          chunk.concat(buffer, es.size)
+
+          begin
+            result = @instance.write(chunk)
+          ensure
+            chunk.purge
+          end
+        }
+        result
+      end
+    end
+
+    class TimeSlicedOutputTestDriver < InputTestDriver
+      def initialize(klass, tag='test', &block)
+        super(klass, &block)
+        @entries = []
+        @expected_buffer = nil
+        @tag = tag
+      end
+
+      attr_accessor :tag
+
+      def emit(record, time=Engine.now)
+        @entries << [time, record]
+        self
+      end
+
+      def expect_format(str)
+        (@expected_buffer ||= '') << str
+      end
+
+      def run(&block)
+        result = []
+        super {
+          block.call if block
+
+          buffer = ''
+          lines = {}
+          # v0.12 TimeSlicedOutput doesn't call #format_stream
+          @entries.each do |time, record|
+            meta = @instance.metadata(@tag, time, record)
+            line = @instance.format(@tag, time, record)
+            buffer << line
+            lines[meta] ||= []
+            lines[meta] << line
+          end
+
+          if @expected_buffer
+            assert_equal(@expected_buffer, buffer)
+          end
+
+          lines.keys.each do |meta|
+            chunk = @instance.buffer.generate_chunk(meta).staged!
+            chunk.append(lines[meta])
+            begin
+              result.push(@instance.write(chunk))
+            ensure
+              chunk.purge
+            end
+          end
+        }
+        result
+      end
+    end
+  end
+end

data/lib/fluent/test/parser_test.rb

@@ -0,0 +1,70 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/parser'
+require 'fluent/config'
+
+module Fluent
+  module Test
+    class ParserTestDriver
+      def initialize(klass_or_str, format=nil, conf={}, &block)
+        if klass_or_str.is_a?(Class)
+          if block
+            # Create new class for test w/ overwritten methods
+            #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass_or_str.name
+            klass_or_str = Class.new(klass_or_str)
+            klass_or_str.define_singleton_method("name") { klass_name }
+            klass_or_str.module_eval(&block)
+          end
+          case klass_or_str.instance_method(:initialize).arity
+          when 0
+            @instance = klass_or_str.new
+          when -2
+            # for RegexpParser
+            @instance = klass_or_str.new(format, conf)
+          end
+        elsif klass_or_str.is_a?(String)
+          @instance = Fluent::Plugin.new_parser(klass_or_str)
+        else
+          @instance = klass_or_str
+        end
+        @config = Config.new
+      end
+
+      attr_reader :instance, :config
+
+      def configure(conf)
+        case conf
+        when Fluent::Config::Element
+          @config = conf
+        when String
+          @config = Config.parse(conf, 'fluent.conf')
+        when Hash
+          @config = Config::Element.new('ROOT', '', conf, [])
+        else
+          raise "Unknown type... #{conf}"
+        end
+        @instance.configure(@config)
+        self
+      end
+
+      def parse(text, &block)
+        @instance.parse(text, &block)
+      end
+    end
+  end
+end