fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin_helper/thread.rb

@@ -0,0 +1,179 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/clock'
+
+module Fluent
+  module PluginHelper
+    module Thread
+      THREAD_DEFAULT_WAIT_SECONDS = 1
+      THREAD_SHUTDOWN_HARD_TIMEOUT_IN_TESTS = 100 # second
+
+      # stop     : mark callback thread as stopped
+      # shutdown : [-]
+      # close    : correct stopped threads
+      # terminate: kill all threads
+
+      attr_reader :_threads # for test driver
+
+      def thread_current_running?
+        # checker for code in callback of thread_create
+        ::Thread.current[:_fluentd_plugin_helper_thread_running] || false
+      end
+
+      def thread_wait_until_start
+        until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && t[:_fluentd_plugin_helper_thread_started] } }
+          sleep 0.1
+        end
+      end
+
+      def thread_wait_until_stop
+        timeout_at = Fluent::Clock.now + THREAD_SHUTDOWN_HARD_TIMEOUT_IN_TESTS
+        until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && !t[:_fluentd_plugin_helper_thread_running] } }
+          break if Fluent::Clock.now > timeout_at
+          sleep 0.1
+        end
+        @_threads_mutex.synchronize{ @_threads.values }.each do |t|
+          if t.alive?
+            puts "going to kill the thread still running: #{t[:_fluentd_plugin_helper_thread_title]}"
+            t.kill rescue nil
+            t[:_fluentd_plugin_helper_thread_running] = false
+          end
+        end
+      end
+
+      # Ruby 2.2.3 or earlier (and all 2.1.x) cause bug about Threading ("Stack consistency error")
+      #  by passing splatted argument to `yield`
+      # https://bugs.ruby-lang.org/issues/11027
+      # We can enable to pass arguments after expire of Ruby 2.1 (& older 2.2.x)
+      # def thread_create(title, *args)
+      #   Thread.new(*args) do |*t_args|
+      #     yield *t_args
+      def thread_create(title)
+        raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
+        raise ArgumentError, "BUG: callback not specified" unless block_given?
+        m = Mutex.new
+        m.lock
+        thread = ::Thread.new do
+          m.lock # run thread after that thread is successfully set into @_threads
+          m.unlock
+          thread_exit = false
+          ::Thread.current[:_fluentd_plugin_helper_thread_title] = title
+          ::Thread.current[:_fluentd_plugin_helper_thread_started] = true
+          ::Thread.current[:_fluentd_plugin_helper_thread_running] = true
+          begin
+            yield
+            thread_exit = true
+          rescue Exception => e
+            log.warn "thread exited by unexpected error", plugin: self.class, title: title, error: e
+            thread_exit = true
+            raise
+          ensure
+            @_threads_mutex.synchronize do
+              @_threads.delete(::Thread.current.object_id)
+            end
+            ::Thread.current[:_fluentd_plugin_helper_thread_running] = false
+            if ::Thread.current.alive? && !thread_exit
+              log.warn "thread doesn't exit correctly (killed or other reason)", plugin: self.class, title: title, thread: ::Thread.current, error: $!
+            end
+          end
+        end
+        thread.abort_on_exception = true
+        @_threads_mutex.synchronize do
+          @_threads[thread.object_id] = thread
+        end
+        m.unlock
+        thread
+      end
+
+      def thread_exist?(title)
+        @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0
+      end
+
+      def thread_started?(title)
+        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t && t[:_fluentd_plugin_helper_thread_started]
+      end
+
+      def thread_running?(title)
+        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t && t[:_fluentd_plugin_helper_thread_running]
+      end
+
+      def initialize
+        super
+        @_threads_mutex = Mutex.new
+        @_threads = {}
+        @_thread_wait_seconds = THREAD_DEFAULT_WAIT_SECONDS
+      end
+
+      def stop
+        super
+        wakeup_threads = []
+        @_threads_mutex.synchronize do
+          @_threads.each_value do |thread|
+            thread[:_fluentd_plugin_helper_thread_running] = false
+            wakeup_threads << thread if thread.alive? && thread.status == "sleep"
+          end
+        end
+        wakeup_threads.each do |thread|
+          if thread.alive?
+            thread.wakeup
+          end
+        end
+      end
+
+      def after_shutdown
+        super
+        wakeup_threads = []
+        @_threads_mutex.synchronize do
+          @_threads.each_value do |thread|
+            wakeup_threads << thread if thread.alive? && thread.status == "sleep"
+          end
+        end
+        wakeup_threads.each do |thread|
+          thread.wakeup if thread.alive?
+        end
+      end
+
+      def close
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          if !thread || thread.join(@_thread_wait_seconds)
+            @_threads_mutex.synchronize{ @_threads.delete(obj_id) }
+          end
+        end
+
+        super
+      end
+
+      def terminate
+        super
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          log.warn "killing existing thread", thread: thread
+          thread.kill if thread
+        end
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          thread.join
+          @_threads_mutex.synchronize{ @_threads.delete(obj_id) }
+        end
+        @_thread_wait_seconds = nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/timer.rb

@@ -0,0 +1,92 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/event_loop'
+require 'set'
+
+module Fluent
+  module PluginHelper
+    module Timer
+      include Fluent::PluginHelper::EventLoop
+
+      # stop     : turn checker into false (callbacks not called anymore)
+      # shutdown : [-]
+      # close    : [-]
+      # terminate: [-]
+
+      attr_reader :_timers # for tests
+
+      # interval: integer/float, repeat: true/false
+      def timer_execute(title, interval, repeat: true, &block)
+        raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
+        raise ArgumentError, "BUG: block not specified for callback" unless block_given?
+        checker = ->(){ @_timer_running }
+        detacher = ->(watcher){ event_loop_detach(watcher) }
+        timer = TimerWatcher.new(title, interval, repeat, log, checker, detacher, &block)
+        @_timers << title
+        event_loop_attach(timer)
+        timer
+      end
+
+      def timer_running?
+        defined?(@_timer_running) && @_timer_running
+      end
+
+      def initialize
+        super
+        @_timers ||= Set.new
+      end
+
+      def start
+        super
+        @_timer_running = true
+      end
+
+      def stop
+        super
+        @_timer_running = false
+      end
+
+      def terminate
+        super
+        @_timers = nil
+      end
+
+      class TimerWatcher < Coolio::TimerWatcher
+        def initialize(title, interval, repeat, log, checker, detacher, &callback)
+          @title = title
+          @callback = callback
+          @repeat = repeat
+          @log = log
+          @checker = checker
+          @detacher = detacher
+          super(interval, repeat)
+        end
+
+        def on_timer
+          @callback.call if @checker.call
+        rescue => e
+          @log.error "Unexpected error raised. Stopping the timer.", title: @title, error: e
+          @log.error_backtrace
+          detach
+          @log.error "Timer detached.", title: @title
+        ensure
+          @detacher.call(self) unless @repeat
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper.rb

@@ -0,0 +1,73 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/event_emitter'
+require 'fluent/plugin_helper/thread'
+require 'fluent/plugin_helper/event_loop'
+require 'fluent/plugin_helper/timer'
+require 'fluent/plugin_helper/child_process'
+require 'fluent/plugin_helper/storage'
+require 'fluent/plugin_helper/parser'
+require 'fluent/plugin_helper/formatter'
+require 'fluent/plugin_helper/http_server'
+require 'fluent/plugin_helper/inject'
+require 'fluent/plugin_helper/extract'
+require 'fluent/plugin_helper/socket'
+require 'fluent/plugin_helper/server'
+require 'fluent/plugin_helper/counter'
+require 'fluent/plugin_helper/retry_state'
+require 'fluent/plugin_helper/record_accessor'
+require 'fluent/plugin_helper/compat_parameters'
+
+module Fluent
+  module PluginHelper
+    module Mixin
+      def self.included(mod)
+        mod.extend(Fluent::PluginHelper)
+      end
+    end
+
+    def self.extended(mod)
+      def mod.inherited(subclass)
+        subclass.module_eval do
+          @_plugin_helpers_list = []
+        end
+      end
+    end
+
+    def helpers_internal(*snake_case_symbols)
+      helper_modules = []
+      snake_case_symbols.each do |name|
+        begin
+          helper_modules << Fluent::PluginHelper.const_get(name.to_s.split('_').map(&:capitalize).join)
+        rescue NameError
+          raise "Unknown plugin helper:#{name}"
+        end
+      end
+      include(*helper_modules)
+    end
+
+    def helpers(*snake_case_symbols)
+      @_plugin_helpers_list ||= []
+      @_plugin_helpers_list.concat(snake_case_symbols)
+      helpers_internal(*snake_case_symbols)
+    end
+
+    def plugin_helpers
+      @_plugin_helpers_list || []
+    end
+  end
+end

data/lib/fluent/plugin_id.rb

@@ -0,0 +1,80 @@
+#
+# Fluent
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'set'
+
+module Fluent
+  module PluginId
+    @@configured_ids = Set.new
+
+    def initialize
+      super
+      @_plugin_root_dir = nil
+    end
+
+    def configure(conf)
+      @id = conf['@id']
+      @_id_configured = !!@id # plugin id is explicitly configured by users (or not)
+      if @id
+        @id = @id.to_s
+        if @@configured_ids.include?(@id) && !plugin_id_for_test?
+          raise Fluent::ConfigError, "Duplicated plugin id `#{@id}`. Check whole configuration and fix it."
+        end
+        @@configured_ids.add(@id)
+      end
+
+      super
+    end
+
+    def plugin_id_for_test?
+      caller_locations.each do |location|
+        # Thread::Backtrace::Location#path returns base filename or absolute path.
+        # #absolute_path returns absolute_path always.
+        # https://bugs.ruby-lang.org/issues/12159
+        if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+          return true
+        end
+      end
+      false
+    end
+
+    def plugin_id_configured?
+      if instance_variable_defined?("@_id_configured")
+        @_id_configured
+      end
+    end
+
+    def plugin_id
+      if instance_variable_defined?("@id")
+        @id || "object:#{object_id.to_s(16)}"
+      else
+        "object:#{object_id.to_s(16)}"
+      end
+    end
+
+    def plugin_root_dir
+      return @_plugin_root_dir if @_plugin_root_dir
+      return nil unless system_config.root_dir
+      return nil unless plugin_id_configured?
+
+      # Fluent::Plugin::Base#fluentd_worker_id
+      dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
+      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      @_plugin_root_dir = dir.freeze
+      dir
+    end
+  end
+end