fluentd222 1.16.2-x86_64-linux

data/lib/fluent/plugin_helper/storage.rb

@@ -0,0 +1,349 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'forwardable'
+
+require 'fluent/plugin'
+require 'fluent/plugin/storage'
+require 'fluent/plugin_helper/timer'
+require 'fluent/config/element'
+require 'fluent/configurable'
+
+module Fluent
+  module PluginHelper
+    module Storage
+      include Fluent::PluginHelper::Timer
+
+      StorageState = Struct.new(:storage, :running)
+
+      def storage_create(usage: '', type: nil, conf: nil, default_type: nil)
+        if conf && conf.respond_to?(:arg) && !conf.arg.empty?
+          usage = conf.arg
+        end
+        if !usage.empty? && usage !~ /^[a-zA-Z][-_.a-zA-Z0-9]*$/
+          raise Fluent::ConfigError, "Argument in <storage ARG> uses invalid characters: '#{usage}'"
+        end
+
+        s = @_storages[usage]
+        if s && s.running
+          return s.storage
+        elsif s
+          # storage is already created, but not loaded / started
+        else # !s
+          type = if type
+                   type
+                 elsif conf && conf.respond_to?(:[])
+                   raise Fluent::ConfigError, "@type is required in <storage>" unless conf['@type']
+                   conf['@type']
+                 elsif default_type
+                   default_type
+                 else
+                   raise ArgumentError, "BUG: both type and conf are not specified"
+                 end
+          storage = Plugin.new_storage(type, parent: self)
+          config = case conf
+                   when Fluent::Config::Element
+                     conf
+                   when Hash
+                     # in code, programmer may use symbols as keys, but Element needs strings
+                     conf = Hash[conf.map{|k,v| [k.to_s, v]}]
+                     Fluent::Config::Element.new('storage', usage, conf, [])
+                   when nil
+                     Fluent::Config::Element.new('storage', usage, {'@type' => type}, [])
+                   else
+                     raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
+                   end
+          storage.configure(config)
+          if @_storages_started
+            storage.start
+          end
+          s = @_storages[usage] = StorageState.new(wrap_instance(storage), false)
+        end
+
+        s.storage
+      end
+
+      module StorageParams
+        include Fluent::Configurable
+        # minimum section definition to instantiate storage plugin instances
+        config_section :storage, required: false, multi: true, param_name: :storage_configs, init: true do
+          config_argument :usage, :string, default: ''
+          config_param    :@type, :string, default: Fluent::Plugin::Storage::DEFAULT_TYPE
+        end
+      end
+
+      def self.included(mod)
+        mod.include StorageParams
+      end
+
+      attr_reader :_storages # for tests
+
+      def initialize
+        super
+        @_storages_started = false
+        @_storages = {} # usage => storage_state
+      end
+
+      def configure(conf)
+        super
+
+        @storage_configs.each do |section|
+          if !section.usage.empty? && section.usage !~ /^[a-zA-Z][-_.a-zA-Z0-9]*$/
+            raise Fluent::ConfigError, "Argument in <storage ARG> uses invalid characters: '#{section.usage}'"
+          end
+          if @_storages[section.usage]
+            raise Fluent::ConfigError, "duplicated storages configured: #{section.usage}"
+          end
+          storage = Plugin.new_storage(section[:@type], parent: self)
+          storage.configure(section.corresponding_config_element)
+          @_storages[section.usage] = StorageState.new(wrap_instance(storage), false)
+        end
+      end
+
+      def start
+        super
+
+        @_storages_started = true
+        @_storages.each_pair do |usage, s|
+          s.storage.start
+          s.storage.load
+
+          if s.storage.autosave && !s.storage.persistent
+            timer_execute(:storage_autosave, s.storage.autosave_interval, repeat: true) do
+              begin
+                s.storage.save
+              rescue => e
+                log.error "plugin storage failed to save its data", usage: usage, type: type, error: e
+              end
+            end
+          end
+          s.running = true
+        end
+      end
+
+      def storage_operate(method_name, &block)
+        @_storages.each_pair do |usage, s|
+          begin
+            block.call(s) if block_given?
+            s.storage.__send__(method_name)
+          rescue => e
+            log.error "unexpected error while #{method_name}", usage: usage, storage: s.storage, error: e
+          end
+        end
+      end
+
+      def stop
+        super
+        # timer stops automatically in super
+        storage_operate(:stop)
+      end
+
+      def before_shutdown
+        storage_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        storage_operate(:shutdown) do |s|
+          s.storage.save if s.storage.save_at_shutdown
+        end
+        super
+      end
+
+      def after_shutdown
+        storage_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        storage_operate(:close){|s| s.running = false }
+        super
+      end
+
+      def terminate
+        storage_operate(:terminate)
+        @_storages = {}
+        super
+      end
+
+      def wrap_instance(storage)
+        if storage.persistent && storage.persistent_always?
+          storage
+        elsif storage.persistent
+          PersistentWrapper.new(storage)
+        elsif !storage.synchronized?
+          SynchronizeWrapper.new(storage)
+        else
+          storage
+        end
+      end
+
+      class PersistentWrapper
+        # PersistentWrapper always provides synchronized operations
+        extend Forwardable
+
+        def initialize(storage)
+          @storage = storage
+          @monitor = Monitor.new
+        end
+
+        def_delegators :@storage, :autosave_interval, :save_at_shutdown
+        def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate
+        def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?
+
+        def method_missing(name, *args)
+          @monitor.synchronize{ @storage.__send__(name, *args) }
+        end
+
+        def persistent_always?
+          true
+        end
+
+        def persistent
+          true
+        end
+
+        def autosave
+          false
+        end
+
+        def synchronized?
+          true
+        end
+
+        def implementation
+          @storage
+        end
+
+        def load
+          @monitor.synchronize do
+            @storage.load
+          end
+        end
+
+        def save
+          @monitor.synchronize do
+            @storage.save
+          end
+        end
+
+        def get(key)
+          @monitor.synchronize do
+            @storage.load
+            @storage.get(key)
+          end
+        end
+
+        def fetch(key, defval)
+          @monitor.synchronize do
+            @storage.load
+            @storage.fetch(key, defval)
+          end
+        end
+
+        def put(key, value)
+          @monitor.synchronize do
+            @storage.load
+            @storage.put(key, value)
+            @storage.save
+            value
+          end
+        end
+
+        def delete(key)
+          @monitor.synchronize do
+            @storage.load
+            val = @storage.delete(key)
+            @storage.save
+            val
+          end
+        end
+
+        def update(key, &block)
+          @monitor.synchronize do
+            @storage.load
+            v = block.call(@storage.get(key))
+            @storage.put(key, v)
+            @storage.save
+            v
+          end
+        end
+      end
+
+      class SynchronizeWrapper
+        extend Forwardable
+
+        def initialize(storage)
+          @storage = storage
+          @monitor = Monitor.new
+        end
+
+        def_delegators :@storage, :persistent, :autosave, :autosave_interval, :save_at_shutdown
+        def_delegators :@storage, :persistent_always?
+        def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate
+        def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?
+
+        def method_missing(name, *args)
+          @monitor.synchronize{ @storage.__send__(name, *args) }
+        end
+
+        def synchronized?
+          true
+        end
+
+        def implementation
+          @storage
+        end
+
+        def load
+          @monitor.synchronize do
+            @storage.load
+          end
+        end
+
+        def save
+          @monitor.synchronize do
+            @storage.save
+          end
+        end
+
+        def get(key)
+          @monitor.synchronize{ @storage.get(key) }
+        end
+
+        def fetch(key, defval)
+          @monitor.synchronize{ @storage.fetch(key, defval) }
+        end
+
+        def put(key, value)
+          @monitor.synchronize{ @storage.put(key, value) }
+        end
+
+        def delete(key)
+          @monitor.synchronize{ @storage.delete(key) }
+        end
+
+        def update(key, &block)
+          @monitor.synchronize do
+            v = block.call(@storage.get(key))
+            @storage.put(key, v)
+            v
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/thread.rb

@@ -0,0 +1,180 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/clock'
+
+module Fluent
+  module PluginHelper
+    module Thread
+      THREAD_DEFAULT_WAIT_SECONDS = 1
+      THREAD_SHUTDOWN_HARD_TIMEOUT_IN_TESTS = 100 # second
+
+      # stop     : mark callback thread as stopped
+      # shutdown : [-]
+      # close    : correct stopped threads
+      # terminate: kill all threads
+
+      attr_reader :_threads # for test driver
+
+      def thread_current_running?
+        # checker for code in callback of thread_create
+        ::Thread.current[:_fluentd_plugin_helper_thread_running] || false
+      end
+
+      def thread_wait_until_start
+        until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && t[:_fluentd_plugin_helper_thread_started] } }
+          sleep 0.1
+        end
+      end
+
+      def thread_wait_until_stop
+        timeout_at = Fluent::Clock.now + THREAD_SHUTDOWN_HARD_TIMEOUT_IN_TESTS
+        until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && !t[:_fluentd_plugin_helper_thread_running] } }
+          break if Fluent::Clock.now > timeout_at
+          sleep 0.1
+        end
+        @_threads_mutex.synchronize{ @_threads.values }.each do |t|
+          if t.alive?
+            puts "going to kill the thread still running: #{t[:_fluentd_plugin_helper_thread_title]}"
+            t.kill rescue nil
+            t[:_fluentd_plugin_helper_thread_running] = false
+          end
+        end
+      end
+
+      # Ruby 2.2.3 or earlier (and all 2.1.x) cause bug about Threading ("Stack consistency error")
+      #  by passing splatted argument to `yield`
+      # https://bugs.ruby-lang.org/issues/11027
+      # We can enable to pass arguments after expire of Ruby 2.1 (& older 2.2.x)
+      # def thread_create(title, *args)
+      #   Thread.new(*args) do |*t_args|
+      #     yield *t_args
+      def thread_create(title)
+        raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
+        raise ArgumentError, "BUG: callback not specified" unless block_given?
+        m = Mutex.new
+        m.lock
+        thread = ::Thread.new do
+          m.lock # run thread after that thread is successfully set into @_threads
+          m.unlock
+          thread_exit = false
+          ::Thread.current[:_fluentd_plugin_helper_thread_title] = title
+          ::Thread.current[:_fluentd_plugin_helper_thread_started] = true
+          ::Thread.current[:_fluentd_plugin_helper_thread_running] = true
+          begin
+            yield
+            thread_exit = true
+          rescue Exception => e
+            log.warn "thread exited by unexpected error", plugin: self.class, title: title, error: e
+            thread_exit = true
+            raise
+          ensure
+            @_threads_mutex.synchronize do
+              @_threads.delete(::Thread.current.object_id)
+            end
+            ::Thread.current[:_fluentd_plugin_helper_thread_running] = false
+            if ::Thread.current.alive? && !thread_exit
+              log.warn "thread doesn't exit correctly (killed or other reason)", plugin: self.class, title: title, thread: ::Thread.current, error: $!
+            end
+          end
+        end
+        thread.abort_on_exception = true
+        thread.name = title.to_s if thread.respond_to?(:name)
+        @_threads_mutex.synchronize do
+          @_threads[thread.object_id] = thread
+        end
+        m.unlock
+        thread
+      end
+
+      def thread_exist?(title)
+        @_threads.values.count{|thread| title == thread[:_fluentd_plugin_helper_thread_title] } > 0
+      end
+
+      def thread_started?(title)
+        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
+        t && t[:_fluentd_plugin_helper_thread_started]
+      end
+
+      def thread_running?(title)
+        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
+        t && t[:_fluentd_plugin_helper_thread_running]
+      end
+
+      def initialize
+        super
+        @_threads_mutex = Mutex.new
+        @_threads = {}
+        @_thread_wait_seconds = THREAD_DEFAULT_WAIT_SECONDS
+      end
+
+      def stop
+        super
+        wakeup_threads = []
+        @_threads_mutex.synchronize do
+          @_threads.each_value do |thread|
+            thread[:_fluentd_plugin_helper_thread_running] = false
+            wakeup_threads << thread if thread.alive? && thread.status == "sleep"
+          end
+        end
+        wakeup_threads.each do |thread|
+          if thread.alive?
+            thread.wakeup
+          end
+        end
+      end
+
+      def after_shutdown
+        super
+        wakeup_threads = []
+        @_threads_mutex.synchronize do
+          @_threads.each_value do |thread|
+            wakeup_threads << thread if thread.alive? && thread.status == "sleep"
+          end
+        end
+        wakeup_threads.each do |thread|
+          thread.wakeup if thread.alive?
+        end
+      end
+
+      def close
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          if !thread || thread.join(@_thread_wait_seconds)
+            @_threads_mutex.synchronize{ @_threads.delete(obj_id) }
+          end
+        end
+
+        super
+      end
+
+      def terminate
+        super
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          log.warn "killing existing thread", thread: thread
+          thread.kill if thread
+        end
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          thread.join
+          @_threads_mutex.synchronize{ @_threads.delete(obj_id) }
+        end
+        @_thread_wait_seconds = nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/timer.rb

@@ -0,0 +1,92 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/event_loop'
+require 'set'
+
+module Fluent
+  module PluginHelper
+    module Timer
+      include Fluent::PluginHelper::EventLoop
+
+      # stop     : turn checker into false (callbacks not called anymore)
+      # shutdown : [-]
+      # close    : [-]
+      # terminate: [-]
+
+      attr_reader :_timers # for tests
+
+      # interval: integer/float, repeat: true/false
+      def timer_execute(title, interval, repeat: true, &block)
+        raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
+        raise ArgumentError, "BUG: block not specified for callback" unless block_given?
+        checker = ->(){ @_timer_running }
+        detacher = ->(watcher){ event_loop_detach(watcher) }
+        timer = TimerWatcher.new(title, interval, repeat, log, checker, detacher, &block)
+        @_timers << title
+        event_loop_attach(timer)
+        timer
+      end
+
+      def timer_running?
+        defined?(@_timer_running) && @_timer_running
+      end
+
+      def initialize
+        super
+        @_timers ||= Set.new
+      end
+
+      def start
+        super
+        @_timer_running = true
+      end
+
+      def stop
+        super
+        @_timer_running = false
+      end
+
+      def terminate
+        super
+        @_timers = nil
+      end
+
+      class TimerWatcher < Coolio::TimerWatcher
+        def initialize(title, interval, repeat, log, checker, detacher, &callback)
+          @title = title
+          @callback = callback
+          @repeat = repeat
+          @log = log
+          @checker = checker
+          @detacher = detacher
+          super(interval, repeat)
+        end
+
+        def on_timer
+          @callback.call if @checker.call
+        rescue => e
+          @log.error "Unexpected error raised. Stopping the timer.", title: @title, error: e
+          @log.error_backtrace
+          detach
+          @log.error "Timer detached.", title: @title
+        ensure
+          @detacher.call(self) unless @repeat
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper.rb

@@ -0,0 +1,75 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/event_emitter'
+require 'fluent/plugin_helper/thread'
+require 'fluent/plugin_helper/event_loop'
+require 'fluent/plugin_helper/timer'
+require 'fluent/plugin_helper/child_process'
+require 'fluent/plugin_helper/storage'
+require 'fluent/plugin_helper/parser'
+require 'fluent/plugin_helper/formatter'
+require 'fluent/plugin_helper/http_server'
+require 'fluent/plugin_helper/inject'
+require 'fluent/plugin_helper/extract'
+require 'fluent/plugin_helper/socket'
+require 'fluent/plugin_helper/server'
+require 'fluent/plugin_helper/counter'
+require 'fluent/plugin_helper/retry_state'
+require 'fluent/plugin_helper/record_accessor'
+require 'fluent/plugin_helper/compat_parameters'
+require 'fluent/plugin_helper/service_discovery'
+require 'fluent/plugin_helper/metrics'
+
+module Fluent
+  module PluginHelper
+    module Mixin
+      def self.included(mod)
+        mod.extend(Fluent::PluginHelper)
+      end
+    end
+
+    def self.extended(mod)
+      def mod.inherited(subclass)
+        subclass.module_eval do
+          @_plugin_helpers_list = []
+        end
+      end
+    end
+
+    def helpers_internal(*snake_case_symbols)
+      helper_modules = []
+      snake_case_symbols.each do |name|
+        begin
+          helper_modules << Fluent::PluginHelper.const_get(name.to_s.split('_').map(&:capitalize).join)
+        rescue NameError
+          raise "Unknown plugin helper:#{name}"
+        end
+      end
+      include(*helper_modules)
+    end
+
+    def helpers(*snake_case_symbols)
+      @_plugin_helpers_list ||= []
+      @_plugin_helpers_list.concat(snake_case_symbols)
+      helpers_internal(*snake_case_symbols)
+    end
+
+    def plugin_helpers
+      @_plugin_helpers_list || []
+    end
+  end
+end