fluentd 0.12.43 → 0.14.0

data/lib/fluent/plugin_helper/storage.rb

@@ -0,0 +1,308 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'monitor'
+require 'forwardable'
+
+require 'fluent/plugin'
+require 'fluent/plugin/storage'
+require 'fluent/plugin_helper/thread'
+require 'fluent/plugin_helper/timer'
+require 'fluent/config/element'
+
+module Fluent
+  module PluginHelper
+    module Storage
+      include Fluent::PluginHelper::Thread
+      include Fluent::PluginHelper::Timer
+
+      StorageState = Struct.new(:storage, :running)
+
+      def storage_create(usage: '', type: nil, conf: nil)
+        s = @_storages[usage]
+        if s && s.running
+          return s.storage
+        elsif !s
+          unless type
+            raise ArgumentError, "BUG: type not specified without configuration"
+          end
+          storage = Plugin.new_storage(type, parent: self)
+          config = case conf
+                   when Fluent::Config::Element
+                     conf
+                   when Hash
+                     conf = Hash[conf.map{|k,v| [k.to_s, v]}]
+                     Fluent::Config::Element.new('storage', '', conf, [])
+                   when nil
+                     Fluent::Config::Element.new('storage', '', {}, [])
+                   else
+                     raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
+                   end
+          storage.configure(config)
+          s = @_storages[usage] = StorageState.new(wrap_instance(storage), false)
+        end
+
+        s.storage.load
+
+        if s.storage.autosave && !s.storage.persistent
+          timer_execute(:storage_autosave, s.storage.autosave_interval, repeat: true) do
+            begin
+              s.storage.save
+            rescue => e
+              log.error "plugin storage failed to save its data", usage: usage, type: type, error: e
+            end
+          end
+        end
+        s.running = true
+        s.storage
+      end
+
+      def self.included(mod)
+        mod.instance_eval do
+          # minimum section definition to instantiate storage plugin instances
+          config_section :storage, required: false, multi: true, param_name: :storage_configs do
+            config_argument :usage, :string, default: ''
+            config_param    :@type, :string, default: Fluent::Plugin::Storage::DEFAULT_TYPE
+          end
+        end
+      end
+
+      attr_reader :_storages # for tests
+
+      def initialize
+        super
+        @_storages = {} # usage => storage_state
+        @_storages_mutex = Mutex.new
+      end
+
+      def configure(conf)
+        super
+
+        @storage_configs.each do |section|
+          if @_storages[section.usage]
+            raise Fluent::ConfigError, "duplicated storages configured: #{section.usage}"
+          end
+          config = conf.elements(name: 'storage', arg: section.usage).first
+          raise "storage section with argument '#{section.usage}' not found. it may be a bug." unless config
+
+          storage = Plugin.new_storage(section[:@type])
+          storage.owner = self
+          storage.configure(config)
+          @_storages[section.usage] = StorageState.new(wrap_instance(storage), false)
+        end
+      end
+
+      def stop
+        super
+        # timer stops automatically
+      end
+
+      def shutdown
+        @_storages.each_pair do |usage, s|
+          begin
+            s.storage.save if s.storage.save_at_shutdown
+          rescue => e
+            log.error "unexpected error while saving data of plugin storages", usage: usage, storage: s.storage, error: e
+          end
+        end
+
+        super
+      end
+
+      def close
+        @_storages.each_pair do |usage, s|
+          begin
+            s.storage.close
+          rescue => e
+            log.error "unexpected error while closing plugin storages", usage: usage, storage: s.storage, error: e
+          end
+          s.running = false
+        end
+
+        super
+      end
+
+      def terminate
+        @_storages.each_pair do |usage, s|
+          begin
+            s.storage.terminate
+          rescue => e
+            log.error "unexpected error while terminating plugin storages", usage: usage, storage: s.storage, error: e
+          end
+        end
+        @_storages = {}
+
+        super
+      end
+
+      def wrap_instance(storage)
+        if storage.persistent && storage.persistent_always?
+          storage
+        elsif storage.persistent
+          PersistentWrapper.new(storage)
+        elsif !storage.synchronized?
+          SynchronizeWrapper.new(storage)
+        else
+          storage
+        end
+      end
+
+      class PersistentWrapper
+        # PersistentWrapper always provides synchronized operations
+        extend Forwardable
+
+        def initialize(storage)
+          @storage = storage
+          @monitor = Monitor.new
+        end
+
+        def_delegators :@storage, :autosave_interval, :save_at_shutdown
+        def_delegators :@storage, :close, :terminate
+
+        def persistent_always?
+          true
+        end
+
+        def persistent
+          true
+        end
+
+        def autosave
+          false
+        end
+
+        def synchronized?
+          true
+        end
+
+        def implementation
+          @storage
+        end
+
+        def load
+          @monitor.synchronize do
+            @storage.load
+          end
+        end
+
+        def save
+          @monitor.synchronize do
+            @storage.save
+          end
+        end
+
+        def get(key)
+          @monitor.synchronize do
+            @storage.load
+            @storage.get(key)
+          end
+        end
+
+        def fetch(key, defval)
+          @monitor.synchronize do
+            @storage.load
+            @storage.fetch(key, defval)
+          end
+        end
+
+        def put(key, value)
+          @monitor.synchronize do
+            @storage.load
+            @storage.put(key, value)
+            @storage.save
+            value
+          end
+        end
+
+        def delete(key)
+          @monitor.synchronize do
+            @storage.load
+            val = @storage.delete(key)
+            @storage.save
+            val
+          end
+        end
+
+        def update(key, &block)
+          @monitor.synchronize do
+            @storage.load
+            v = block.call(@storage.get(key))
+            @storage.put(key, v)
+            @storage.save
+            v
+          end
+        end
+      end
+
+      class SynchronizeWrapper
+        extend Forwardable
+
+        def initialize(storage)
+          @storage = storage
+          @mutex = Mutex.new
+        end
+
+        def_delegators :@storage, :persistent, :autosave, :autosave_interval, :save_at_shutdown
+        def_delegators :@storage, :persistent_always?
+        def_delegators :@storage, :close, :terminate
+
+        def synchronized?
+          true
+        end
+
+        def implementation
+          @storage
+        end
+
+        def load
+          @mutex.synchronize do
+            @storage.load
+          end
+        end
+
+        def save
+          @mutex.synchronize do
+            @storage.save
+          end
+        end
+
+        def get(key)
+          @mutex.synchronize{ @storage.get(key) }
+        end
+
+        def fetch(key, defval)
+          @mutex.synchronize{ @storage.fetch(key, defval) }
+        end
+
+        def put(key, value)
+          @mutex.synchronize{ @storage.put(key, value) }
+        end
+
+        def delete(key)
+          @mutex.synchronize{ @storage.delete(key) }
+        end
+
+        def update(key, &block)
+          @mutex.synchronize do
+            v = block.call(@storage.get(key))
+            @storage.put(key, v)
+            v
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/thread.rb

@@ -0,0 +1,147 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module Thread
+      THREAD_DEFAULT_WAIT_SECONDS = 1
+
+      # stop     : mark callback thread as stopped
+      # shutdown : [-]
+      # close    : correct stopped threads
+      # terminate: kill all threads
+
+      attr_reader :_threads # for test driver
+
+      def thread_current_running?
+        # checker for code in callback of thread_create
+        ::Thread.current[:_fluentd_plugin_helper_thread_running] || false
+      end
+
+      def thread_wait_until_start
+        until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && t[:_fluentd_plugin_helper_thread_started] } }
+          sleep 0.1
+        end
+      end
+
+      def thread_wait_until_stop
+        until @_threads_mutex.synchronize{ @_threads.values.reduce(true){|r,t| r && ![:_fluentd_plugin_helper_thread_running] } }
+          sleep 0.1
+        end
+      end
+
+      # Ruby 2.2.3 or earlier (and all 2.1.x) cause bug about Threading ("Stack consistency error")
+      #  by passing splatted argument to `yield`
+      # https://bugs.ruby-lang.org/issues/11027
+      # We can enable to pass arguments after expire of Ruby 2.1 (& older 2.2.x)
+      # def thread_create(title, *args)
+      #   Thread.new(*args) do |*t_args|
+      #     yield *t_args
+      def thread_create(title)
+        raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
+        raise ArgumentError, "BUG: callback not specified" unless block_given?
+        m = Mutex.new
+        m.lock
+        thread = ::Thread.new do
+          m.lock # run thread after that thread is successfully set into @_threads
+          m.unlock
+          thread_exit = false
+          ::Thread.current[:_fluentd_plugin_helper_thread_title] = title
+          ::Thread.current[:_fluentd_plugin_helper_thread_started] = true
+          ::Thread.current[:_fluentd_plugin_helper_thread_running] = true
+          begin
+            yield
+            thread_exit = true
+          rescue Exception => e
+            log.warn "thread exited by unexpected error", plugin: self.class, title: title, error: e
+            thread_exit = true
+            raise
+          ensure
+            unless thread_exit
+              log.warn "thread doesn't exit correctly (killed or other reason)", plugin: self.class, title: title, thread: ::Thread.current, error: $!
+            end
+            @_threads_mutex.synchronize do
+              @_threads.delete(::Thread.current.object_id)
+            end
+            ::Thread.current[:_fluentd_plugin_helper_thread_running] = false
+          end
+        end
+        thread.abort_on_exception = true
+        @_threads_mutex.synchronize do
+          @_threads[thread.object_id] = thread
+        end
+        m.unlock
+        thread
+      end
+
+      def thread_exist?(title)
+        @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0
+      end
+
+      def thread_started?(title)
+        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t && t[:_fluentd_plugin_helper_thread_started]
+      end
+
+      def thread_running?(title)
+        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t && t[:_fluentd_plugin_helper_thread_running]
+      end
+
+      def initialize
+        super
+        @_threads_mutex = Mutex.new
+        @_threads = {}
+        @_thread_wait_seconds = THREAD_DEFAULT_WAIT_SECONDS
+      end
+
+      def stop
+        super
+        @_threads_mutex.synchronize do
+          @_threads.each_pair do |obj_id, thread|
+            thread[:_fluentd_plugin_helper_thread_running] = false
+          end
+        end
+      end
+
+      def close
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          if !thread || thread.join(@_thread_wait_seconds)
+            @_threads_mutex.synchronize{ @_threads.delete(obj_id) }
+          end
+        end
+
+        super
+      end
+
+      def terminate
+        super
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          log.warn "killing existing thead", thread: thread
+          thread.kill if thread
+        end
+        @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
+          thread = @_threads[obj_id]
+          thread.join
+          @_threads_mutex.synchronize{ @_threads.delete(obj_id) }
+        end
+        @_thread_wait_seconds = nil
+      end
+    end
+  end
+end