fluentd-hubspot 0.14.14.1

data/lib/fluent/test/driver/output.rb

@@ -0,0 +1,102 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owner'
+require 'fluent/test/driver/event_feeder'
+
+require 'fluent/plugin/output'
+require 'timeout'
+
+module Fluent
+  module Test
+    module Driver
+      class Output < BaseOwner
+        include EventFeeder
+
+        def initialize(klass, opts: {}, &block)
+          super
+          raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Output" unless @instance.is_a? Fluent::Plugin::Output
+          @flush_buffer_at_cleanup = nil
+          @wait_flush_completion = nil
+          @force_flush_retry = nil
+          @format_hook = nil
+          @format_results = []
+        end
+
+        def run(flush: true, wait_flush_completion: true, force_flush_retry: false, **kwargs, &block)
+          @flush_buffer_at_cleanup = flush
+          @wait_flush_completion = wait_flush_completion
+          @force_flush_retry = force_flush_retry
+          super(**kwargs, &block)
+        end
+
+        def run_actual(**kwargs, &block)
+          if @force_flush_retry
+            @instance.retry_for_error_chunk = true
+          end
+          val = super(**kwargs, &block)
+          if @flush_buffer_at_cleanup
+            self.flush
+          end
+          val
+        ensure
+          @instance.retry_for_error_chunk = false
+        end
+
+        def formatted
+          @format_results
+        end
+
+        def flush
+          @instance.force_flush
+          wait_flush_completion if @wait_flush_completion
+        end
+
+        def wait_flush_completion
+          buffer_queue = ->(){ @instance.buffer && @instance.buffer.queue.size > 0 }
+          dequeued_chunks = ->(){
+            @instance.dequeued_chunks_mutex &&
+            @instance.dequeued_chunks &&
+            @instance.dequeued_chunks_mutex.synchronize{ @instance.dequeued_chunks.size > 0 }
+          }
+
+          Timeout.timeout(10) do
+            while buffer_queue.call || dequeued_chunks.call
+              sleep 0.1
+            end
+          end
+        end
+
+        def instance_hook_after_started
+          super
+
+          # it's decided after #start whether output plugin instances use @custom_format or not.
+          if @instance.instance_eval{ @custom_format }
+            @format_hook = format_hook = ->(result){ @format_results << result }
+            m = Module.new do
+              define_method(:format) do |tag, time, record|
+                result = super(tag, time, record)
+                format_hook.call(result)
+                result
+              end
+            end
+            @instance.singleton_class.prepend m
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/parser.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/driver/base_owned'
+
+module Fluent
+  module Test
+    module Driver
+      class Parser < BaseOwned
+        def initialize(klass, **kwargs, &block)
+          super
+          @section_name = "parse"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/driver/test_event_router.rb

@@ -0,0 +1,45 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+
+module Fluent
+  module Test
+    module Driver
+      class TestEventRouter
+        def initialize(driver)
+          @driver = driver
+        end
+
+        def emit(tag, time, record)
+          @driver.emit_event_stream(tag, OneEventStream.new(time, record))
+        end
+
+        def emit_array(tag, array)
+          @driver.emit_event_stream(tag, ArrayEventStream.new(array))
+        end
+
+        def emit_stream(tag, es)
+          @driver.emit_event_stream(tag, es)
+        end
+
+        def emit_error_event(tag, time, record, error)
+          @driver.emit_error_event(tag, time, record, error)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/test/filter_test.rb

@@ -0,0 +1,77 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/test/base'
+require 'fluent/event'
+
+module Fluent
+  module Test
+    class FilterTestDriver < TestDriver
+      def initialize(klass, tag = 'filter.test', &block)
+        super(klass, &block)
+        @tag = tag
+        @events = {}
+        @filtered = MultiEventStream.new
+      end
+
+      attr_reader :filtered
+      attr_accessor :tag
+
+      def emit(record, time = Engine.now)
+        emit_with_tag(@tag, record, time)
+      end
+      alias_method :filter, :emit
+
+      def emit_with_tag(tag, record, time = Engine.now)
+        @events[tag] ||= MultiEventStream.new
+        @events[tag].add(time, record)
+      end
+      alias_method :filter_with_tag, :emit_with_tag
+
+      def filter_stream(es)
+        filter_stream_with_tag(@tag, es)
+      end
+
+      def filter_stream_with_tag(tag, es)
+        @events[tag] = es
+      end
+
+      def filtered_as_array
+        all = []
+        @filtered.each { |time, record|
+          all << [@tag, time, record]
+        }
+        all
+      end
+      alias_method :emits, :filtered_as_array # emits is for consistent with other drivers
+
+      # Almost filters don't use threads so default is 0. It reduces test time.
+      def run(num_waits = 0, &block)
+        super(num_waits) {
+          block.call if block
+
+          @events.each { |tag, es|
+            processed = @instance.filter_stream(tag, es)
+            processed.each { |time, record|
+              @filtered.add(time, record)
+            }
+          }
+        }
+        self
+      end
+    end
+  end
+end

data/lib/fluent/test/formatter_test.rb

@@ -0,0 +1,65 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/formatter'
+require 'fluent/config'
+require 'fluent/plugin'
+
+module Fluent
+  module Test
+    class FormatterTestDriver
+      def initialize(klass_or_str, proc=nil, &block)
+        if klass_or_str.is_a?(Class)
+          if block
+            # Create new class for test w/ overwritten methods
+            #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_name = klass_or_str.name
+            klass_or_str = Class.new(klass_or_str)
+            klass_or_str.define_singleton_method("name") { klass_name }
+            klass_or_str.module_eval(&block)
+          end
+          @instance = klass_or_str.new
+        elsif klass_or_str.is_a?(String)
+          @instance = Fluent::Plugin.new_formatter(klass_or_str)
+        else
+          @instance = klass_or_str
+        end
+        @config = Config.new
+      end
+
+      attr_reader :instance, :config
+
+      def configure(conf)
+        case conf
+        when Fluent::Config::Element
+          @config = conf
+        when String
+          @config = Config.parse(conf, 'fluent.conf')
+        when Hash
+          @config = Config::Element.new('ROOT', '', conf, [])
+        else
+          raise "Unknown type... #{conf}"
+        end
+        @instance.configure(@config)
+        self
+      end
+
+      def format(tag, time, record)
+        @instance.format(tag, time, record)
+      end
+    end
+  end
+end

data/lib/fluent/test/helpers.rb

@@ -0,0 +1,134 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/element'
+require 'fluent/msgpack_factory'
+require 'fluent/time'
+
+module Fluent
+  module Test
+    module Helpers
+      # See "Example Custom Assertion: http://test-unit.github.io/test-unit/en/Test/Unit/Assertions.html
+      def assert_equal_event_time(expected, actual, message = nil)
+        expected_s = "#{Time.at(expected.sec)} (nsec #{expected.nsec})"
+        actual_s   = "#{Time.at(actual.sec)  } (nsec #{actual.nsec})"
+        message = build_message(message, <<EOT, expected_s, actual_s)
+<?> expected but was
+<?>.
+EOT
+        assert_block(message) do
+          expected.is_a?(Fluent::EventTime) && actual.is_a?(Fluent::EventTime) && expected.sec == actual.sec && expected.nsec == actual.nsec
+        end
+      end
+
+      def config_element(name = 'test', argument = '', params = {}, elements = [])
+        Fluent::Config::Element.new(name, argument, params, elements)
+      end
+
+      def event_time(str=nil, format: nil)
+        if str
+          if format
+            Fluent::EventTime.from_time(Time.strptime(str, format))
+          else
+            Fluent::EventTime.parse(str)
+          end
+        else
+          Fluent::EventTime.now
+        end
+      end
+
+      def with_timezone(tz)
+        oldtz, ENV['TZ'] = ENV['TZ'], tz
+        yield
+      ensure
+        ENV['TZ'] = oldtz
+      end
+
+      def with_worker_config(root_dir: nil, workers: nil, worker_id: nil, &block)
+        if workers
+          if worker_id
+            if worker_id >= workers
+              raise "worker_id must be between 0 and (workers - 1)"
+            end
+          else
+            worker_id = 0
+          end
+        end
+
+        opts = {}
+        opts['root_dir'] = root_dir if root_dir
+        opts['workers'] = workers if workers
+
+        ENV['SERVERENGINE_WORKER_ID'] = worker_id.to_s
+        Fluent::SystemConfig.overwrite_system_config(opts, &block)
+      ensure
+        ENV.delete('SERVERENGINE_WORKER_ID')
+      end
+
+      def time2str(time, localtime: false, format: nil)
+        if format
+          if localtime
+            Time.at(time).strftime(format)
+          else
+            Time.at(time).utc.strftime(format)
+          end
+        else
+          if localtime
+            Time.at(time).iso8601
+          else
+            Time.at(time).utc.iso8601
+          end
+        end
+      end
+
+      def msgpack(type)
+        case type
+        when :factory
+          Fluent::MessagePackFactory.factory
+        when :packer
+          Fluent::MessagePackFactory.packer
+        when :unpacker
+          Fluent::MessagePackFactory.unpacker
+        else
+          raise ArgumentError, "unknown msgpack object type '#{type}'"
+        end
+      end
+
+      #
+      # Use this method with v0.12 compatibility layer.
+      #
+      # For v0.14 API, use `driver.logs` instead.
+      #
+      def capture_log(driver)
+        tmp = driver.instance.log.out
+        driver.instance.log.out = StringIO.new
+        yield
+        return driver.instance.log.out.string
+      ensure
+        driver.instance.log.out = tmp
+      end
+
+      def capture_stdout
+        out = StringIO.new
+        $stdout = out
+        yield
+        out.string.force_encoding('utf-8')
+      ensure
+        $stdout = STDOUT
+      end
+    end
+  end
+end