fluentd-hubspot 0.14.14.1

data/lib/fluent/plugin_helper/thread.rb

@@ -0,0 +1,179 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/clock'
+
+module Fluent
+  module PluginHelper
+    module Thread
+      THREAD_DEFAULT_WAIT_SECONDS = 1
+      THREAD_SHUTDOWN_HARD_TIMEOUT_IN_TESTS = 100 # second
+
+      # stop     : mark callback thread as stopped
+      # shutdown : [-]
+      # close    : correct stopped threads
+      # terminate: kill all threads
+
+      attr_reader :_threads # for test driver
+
+      def thread_current_running?
+        # checker for code in callback of thread_create
+        ::Thread.current[:_fluentd_plugin_helper_thread_running] || false
+      end
+
+      def thread_wait_until_start
+        until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && t[:_fluentd_plugin_helper_thread_started] } }
+          sleep 0.1
+        end
+      end
+
+      def thread_wait_until_stop
+        timeout_at = Fluent::Clock.now + THREAD_SHUTDOWN_HARD_TIMEOUT_IN_TESTS
+        until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && !t[:_fluentd_plugin_helper_thread_running] } }
+          break if Fluent::Clock.now > timeout_at
+          sleep 0.1
+        end
+        @_threads_mutex.synchronize{ @_threads.values }.each do |t|
+          if t.alive?
+            puts "going to kill the thread still running: #{t[:_fluentd_plugin_helper_thread_title]}"
+            t.kill rescue nil
+            t[:_fluentd_plugin_helper_thread_running] = false
+          end
+        end
+      end
+
+      # Ruby 2.2.3 or earlier (and all 2.1.x) cause bug about Threading ("Stack consistency error")
+      #  by passing splatted argument to `yield`
+      # https://bugs.ruby-lang.org/issues/11027
+      # We can enable to pass arguments after expire of Ruby 2.1 (& older 2.2.x)
+      # def thread_create(title, *args)
+      #   Thread.new(*args) do |*t_args|
+      #     yield *t_args
+      def thread_create(title)
+        raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
+        raise ArgumentError, "BUG: callback not specified" unless block_given?
+        m = Mutex.new
+        m.lock
+        thread = ::Thread.new do
+          m.lock # run thread after that thread is successfully set into @_threads
+          m.unlock
+          thread_exit = false
+          ::Thread.current[:_fluentd_plugin_helper_thread_title] = title
+          ::Thread.current[:_fluentd_plugin_helper_thread_started] = true
+          ::Thread.current[:_fluentd_plugin_helper_thread_running] = true
+          begin
+            yield
+            thread_exit = true
+          rescue Exception => e
+            log.warn "thread exited by unexpected error", plugin: self.class, title: title, error: e
+            thread_exit = true
+            raise
+          ensure
+            @_threads_mutex.synchronize do
+              @_threads.delete(::Thread.current.object_id)
+            end
+            ::Thread.current[:_fluentd_plugin_helper_thread_running] = false
+            if ::Thread.current.alive? && !thread_exit
+              log.warn "thread doesn't exit correctly (killed or other reason)", plugin: self.class, title: title, thread: ::Thread.current, error: $!
+            end
+          end
+        end
+        thread.abort_on_exception = true
+        @_threads_mutex.synchronize do
+          @_threads[thread.object_id] = thread
+        end
+        m.unlock
+        thread
+      end
+
+      def thread_exist?(title)
+        @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0
+      end
+
+      def thread_started?(title)
+        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t && t[:_fluentd_plugin_helper_thread_started]
+      end
+
+      def thread_running?(title)
+        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t && t[:_fluentd_plugin_helper_thread_running]
+      end
+
+      def initialize
+        super
+        @_threads_mutex = Mutex.new
+        @_threads = {}
+        @_thread_wait_seconds = THREAD_DEFAULT_WAIT_SECONDS
+      end
+
+      def stop
+        super
+        wakeup_threads = []
+        @_threads_mutex.synchronize do
+          @_threads.each_value do |thread|
+            thread[:_fluentd_plugin_helper_thread_running] = false
+            wakeup_threads << thread if thread.alive? && thread.status == "sleep"
+          end
+        end
+        wakeup_threads.each do |thread|
+          if thread.alive?
+            thread.wakeup
+          end
+        end
+      end
+
+      def after_shutdown
+        super
+        wakeup_threads = []
+        @_threads_mutex.synchronize do
+          @_threads.each_value do |thread|
+            wakeup_threads << thread if thread.alive? && thread.status == "sleep"
+          end
+        end
+        wakeup_threads.each do |thread|
+          thread.wakeup if thread.alive?
+        end
+      end
+
+      def close
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          if !thread || thread.join(@_thread_wait_seconds)
+            @_threads_mutex.synchronize{ @_threads.delete(obj_id) }
+          end
+        end
+
+        super
+      end
+
+      def terminate
+        super
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          log.warn "killing existing thread", thread: thread
+          thread.kill if thread
+        end
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          thread.join
+          @_threads_mutex.synchronize{ @_threads.delete(obj_id) }
+        end
+        @_thread_wait_seconds = nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/timer.rb

@@ -0,0 +1,91 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/event_loop'
+
+module Fluent
+  module PluginHelper
+    module Timer
+      include Fluent::PluginHelper::EventLoop
+
+      # stop     : turn checker into false (callbacks not called anymore)
+      # shutdown : [-]
+      # close    : [-]
+      # terminate: [-]
+
+      attr_reader :_timers # for tests
+
+      # interval: integer/float, repeat: true/false
+      def timer_execute(title, interval, repeat: true, &block)
+        raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
+        raise ArgumentError, "BUG: block not specified for callback" unless block_given?
+        checker = ->(){ @_timer_running }
+        timer = TimerWatcher.new(title, interval, repeat, log, checker, &block)
+        @_timers << title
+        event_loop_attach(timer)
+        timer
+      end
+
+      def timer_running?
+        defined?(@_timer_running) && @_timer_running
+      end
+
+      def initialize
+        super
+        @_timers = []
+      end
+
+      def start
+        super
+        @_timer_running = true
+      end
+
+      def stop
+        super
+        @_timer_running = false
+      end
+
+      def terminate
+        super
+        @_timers = []
+      end
+
+      class TimerWatcher < Coolio::TimerWatcher
+        def initialize(title, interval, repeat, log, checker, &callback)
+          @title = title
+          @callback = callback
+          @repeat = repeat
+          @log = log
+          @checker = checker
+          super(interval, repeat)
+        end
+
+        def on_timer
+          @callback.call if @checker.call
+        rescue => e
+          @log.error "Unexpected error raised. Stopping the timer.", title: @title, error: e
+          @log.error_backtrace
+          detach
+          @log.error "Timer detached.", title: @title
+        ensure
+          if attached?
+            detach unless @repeat
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_id.rb

@@ -0,0 +1,80 @@
+#
+# Fluent
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'set'
+
+module Fluent
+  module PluginId
+    @@configured_ids = Set.new
+
+    def initialize
+      super
+      @_plugin_root_dir = nil
+    end
+
+    def configure(conf)
+      @id = conf['@id']
+      @_id_configured = !!@id # plugin id is explicitly configured by users (or not)
+      if @id
+        @id = @id.to_s
+        if @@configured_ids.include?(@id) && !plugin_id_for_test?
+          raise Fluent::ConfigError, "Duplicated plugin id `#{@id}`. Check whole configuration and fix it."
+        end
+        @@configured_ids.add(@id)
+      end
+
+      super
+    end
+
+    def plugin_id_for_test?
+      caller_locations.each do |location|
+        # Thread::Backtrace::Location#path returns base filename or absolute path.
+        # #absolute_path returns absolute_path always.
+        # https://bugs.ruby-lang.org/issues/12159
+        if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+          return true
+        end
+      end
+      false
+    end
+
+    def plugin_id_configured?
+      if instance_variable_defined?("@_id_configured")
+        @_id_configured
+      end
+    end
+
+    def plugin_id
+      if instance_variable_defined?("@id")
+        @id || "object:#{object_id.to_s(16)}"
+      else
+        "object:#{object_id.to_s(16)}"
+      end
+    end
+
+    def plugin_root_dir
+      return @_plugin_root_dir if @_plugin_root_dir
+      return nil unless system_config.root_dir
+      return nil unless plugin_id_configured?
+
+      # Fluent::Plugin::Base#fluentd_worker_id
+      dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
+      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      @_plugin_root_dir = dir.freeze
+      dir
+    end
+  end
+end

data/lib/fluent/process.rb

@@ -0,0 +1,22 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/detach_process_mixin'
+
+module Fluent
+  DetachProcessMixin = Fluent::Compat::DetachProcessMixin
+  DetachMultiProcessMixin = Fluent::Compat::DetachMultiProcessMixin
+end

data/lib/fluent/registry.rb

@@ -0,0 +1,116 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+
+module Fluent
+  class Registry
+    DEFAULT_PLUGIN_PATH = File.expand_path('../plugin', __FILE__)
+    FLUENT_LIB_PATH = File.dirname(File.dirname(DEFAULT_PLUGIN_PATH))
+
+    def initialize(kind, search_prefix, dir_search_prefix: nil)
+      @kind = kind
+      @search_prefix = search_prefix
+      @dir_search_prefix = dir_search_prefix
+      @map = {}
+      @paths = []
+    end
+
+    attr_reader :kind, :paths, :map, :dir_search_prefix
+
+    def register(type, value)
+      type = type.to_sym
+      @map[type] = value
+    end
+
+    def lookup(type)
+      type = type.to_sym
+      if value = @map[type]
+        return value
+      end
+      search(type)
+      if value = @map[type]
+        return value
+      end
+      raise ConfigError, "Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"  # TODO error class
+    end
+
+    def reverse_lookup(value)
+      @map.each do |k, v|
+        return k if v == value
+      end
+      nil
+    end
+
+    def search(type)
+      # search from additional plugin directories
+      if @dir_search_prefix
+        path = "#{@dir_search_prefix}#{type}"
+        files = @paths.map { |lp|
+          lpath = File.expand_path(File.join(lp, "#{path}.rb"))
+          File.exist?(lpath) ? lpath : nil
+        }.compact
+        unless files.empty?
+          # prefer newer version
+          require files.sort.last
+          return
+        end
+      end
+
+      path = "#{@search_prefix}#{type}"
+
+      # prefer LOAD_PATH than gems
+      files = $LOAD_PATH.map { |lp|
+        if lp == FLUENT_LIB_PATH
+          nil
+        else
+          lpath = File.expand_path(File.join(lp, "#{path}.rb"))
+          File.exist?(lpath) ? lpath : nil
+        end
+      }.compact
+      unless files.empty?
+        # prefer newer version
+        require files.sort.last
+        return
+      end
+
+      # Find from gems and prefer newer version
+      specs = Gem::Specification.find_all { |spec|
+        if spec.name == 'fluentd'.freeze
+          false
+        else
+          spec.contains_requirable_file? path
+        end
+      }.sort_by { |spec| spec.version }
+      if spec = specs.last
+        spec.require_paths.each { |lib|
+          file = "#{spec.full_gem_path}/#{lib}/#{path}"
+          if File.exist?("#{file}.rb")
+            require file
+            return
+          end
+        }
+      end
+
+      # Lastly, load built-in plugin
+      lpath = File.expand_path(File.join(FLUENT_LIB_PATH, "#{@search_prefix}#{type}.rb"))
+      if File.exist?(lpath)
+        require lpath
+        return
+      end
+    end
+  end
+end