fluentd 0.14.0 → 0.14.1

data/lib/fluent/plugin_helper/inject.rb

@@ -0,0 +1,112 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+require 'time'
+
+module Fluent
+  module PluginHelper
+    module Inject
+      def inject_values_to_record(tag, time, record)
+        return record unless @_inject_enabled
+
+        r = record.dup
+        if @_inject_hostname_key
+          r[@_inject_hostname_key] = @_inject_hostname
+        end
+        if @_inject_tag_key
+          r[@_inject_tag_key] = tag
+        end
+        if @_inject_time_key
+          r[@_inject_time_key] = @_inject_time_formatter.call(time)
+        end
+
+        r
+      end
+
+      def inject_values_to_event_stream(tag, es)
+        return es unless @_inject_enabled
+
+        new_es = Fluent::MultiEventStream.new
+        es.each do |time, record|
+          r = record.dup
+          if @_inject_hostname_key
+            r[@_inject_hostname_key] = @_inject_hostname
+          end
+          if @_inject_tag_key
+            r[@_inject_tag_key] = tag
+          end
+          if @_inject_time_key
+            r[@_inject_time_key] = @_inject_time_formatter.call(time)
+          end
+          new_es.add(time, r)
+        end
+
+        new_es
+      end
+
+      def self.included(mod)
+        mod.instance_eval do
+          config_section :inject, required: false, multi: false, param_name: :inject_config do
+            config_param :hostname_key, :string, default: nil
+            config_param :hostname, :string, default: nil
+            config_param :tag_key, :string, default: nil
+            config_param :time_key, :string, default: nil
+            config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+            config_param :time_format, :string, default: nil
+            config_param :timezone, :string, default: "#{Time.now.strftime('%z')}" # localtime
+          end
+        end
+      end
+
+      def initialize
+        super
+        @_inject_hostname_key = nil
+        @_inject_hostname = nil
+        @_inject_tag_key = nil
+        @_inject_time_key = nil
+        @_inject_time_formatter = nil
+      end
+
+      def configure(conf)
+        super
+
+        if @inject_config
+          @_inject_hostname_key = @inject_config.hostname_key
+          if @_inject_hostname_key
+            @_inject_hostname =  @inject_config.hostname
+            unless @_inject_hostname
+              @_inject_hostname = Socket.gethostname
+              log.info "using hostname for specified field", host_key: @_inject_hostname_key, host_name: @_inject_hostname
+            end
+          end
+          @_inject_tag_key = @inject_config.tag_key
+          @_inject_time_key = @inject_config.time_key
+          if @_inject_time_key
+            @_inject_time_formatter = case @inject_config.time_type
+                                      when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
+                                      when :unixtime then ->(time){ time.to_i }
+                                      else
+                                        Fluent::TimeFormatter.new(@inject_config.time_format, false, @inject_config.timezone)
+                                      end
+          end
+
+          @_inject_enabled = @_inject_hostname_key || @_inject_tag_key || @_inject_time_key
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/parser.rb

@@ -0,0 +1,138 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/parser'
+require 'fluent/config/element'
+
+module Fluent
+  module PluginHelper
+    module Parser
+      def parser_create(usage: '', type: nil, conf: nil)
+        parser = @_parsers[usage]
+        return parser if parser
+
+        if !type
+          raise ArgumentError, "BUG: both type and conf are not specified" unless conf
+          raise Fluent::ConfigError, "@type is required in <parse>" unless conf['@type']
+          type = conf['@type']
+        end
+        parser = Fluent::Plugin.new_parser(type, parent: self)
+        config = case conf
+                 when Fluent::Config::Element
+                   conf
+                 when Hash
+                   # in code, programmer may use symbols as keys, but Element needs strings
+                   conf = Hash[conf.map{|k,v| [k.to_s, v]}]
+                   Fluent::Config::Element.new('parse', usage, conf, [])
+                 when nil
+                   Fluent::Config::Element.new('parse', usage, {}, [])
+                 else
+                   raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
+                 end
+        parser.configure(config)
+        if @_parsers_started
+          parser.start
+        end
+
+        @_parsers[usage] = parser
+        parser
+      end
+
+      def self.included(mod)
+        mod.instance_eval do
+          # minimum section definition to instantiate parser plugin instances
+          config_section :parse, required: false, multi: true, param_name: :parser_configs do
+            config_argument :usage, :string, default: ''
+            config_param    :@type, :string
+          end
+        end
+      end
+
+      attr_reader :_parsers # for tests
+
+      def initialize
+        super
+        @_parsers_started = false
+        @_parsers = {} # usage => parser
+      end
+
+      def configure(conf)
+        super
+
+        @parser_configs.each do |section|
+          if @_parsers[section.usage]
+            raise Fluent::ConfigError, "duplicated parsers configured: #{section.usage}"
+          end
+          parser = Plugin.new_parser(section[:@type], parent: self)
+          parser.configure(section.corresponding_config_element)
+          @_parsers[section.usage] = parser
+        end
+      end
+
+      def start
+        super
+        @_parsers_started = true
+        @_parsers.each_pair do |usage, parser|
+          parser.start
+        end
+      end
+
+      def parser_operate(method_name, &block)
+        @_parsers.each_pair do |usage, parser|
+          begin
+            parser.send(method_name)
+            block.call(parser) if block_given?
+          rescue => e
+            log.error "unexpected error while #{method_name}", usage: usage, parser: parser, error: e
+          end
+        end
+      end
+
+      def stop
+        super
+        parser_operate(:stop)
+      end
+
+      def before_shutdown
+        parser_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        parser_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        parser_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        parser_operate(:close)
+        super
+      end
+
+      def terminate
+        parser_operate(:terminate)
+        @_parsers_started = false
+        @_parsers = {}
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/storage.rb

@@ -14,19 +14,16 @@
 #    limitations under the License.
 #
 
-require 'monitor'
 require 'forwardable'
 
 require 'fluent/plugin'
 require 'fluent/plugin/storage'
-require 'fluent/plugin_helper/thread'
 require 'fluent/plugin_helper/timer'
 require 'fluent/config/element'
 
 module Fluent
   module PluginHelper
     module Storage
-      include Fluent::PluginHelper::Thread
       include Fluent::PluginHelper::Timer
 
       StorageState = Struct.new(:storage, :running)
@@ -35,38 +32,34 @@ module Fluent
         s = @_storages[usage]
         if s && s.running
           return s.storage
-        elsif !s
-          unless type
-            raise ArgumentError, "BUG: type not specified without configuration"
+        elsif s
+          # storage is already created, but not loaded / started
+        else # !s
+          if !type
+            raise ArgumentError, "BUG: both type and conf are not specified" unless conf
+            raise Fluent::ConfigError, "@type is not specified for <storage>" unless conf['@type']
+            type = conf['@type']
           end
           storage = Plugin.new_storage(type, parent: self)
           config = case conf
                    when Fluent::Config::Element
                      conf
                    when Hash
+                     # in code, programmer may use symbols as keys, but Element needs strings
                      conf = Hash[conf.map{|k,v| [k.to_s, v]}]
-                     Fluent::Config::Element.new('storage', '', conf, [])
+                     Fluent::Config::Element.new('storage', usage, conf, [])
                    when nil
-                     Fluent::Config::Element.new('storage', '', {}, [])
+                     Fluent::Config::Element.new('storage', usage, {}, [])
                    else
                      raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
                    end
           storage.configure(config)
+          if @_storages_started
+            storage.start
+          end
           s = @_storages[usage] = StorageState.new(wrap_instance(storage), false)
         end
 
-        s.storage.load
-
-        if s.storage.autosave && !s.storage.persistent
-          timer_execute(:storage_autosave, s.storage.autosave_interval, repeat: true) do
-            begin
-              s.storage.save
-            rescue => e
-              log.error "plugin storage failed to save its data", usage: usage, type: type, error: e
-            end
-          end
-        end
-        s.running = true
         s.storage
       end
 
@@ -84,8 +77,8 @@ module Fluent
 
       def initialize
         super
+        @_storages_started = false
         @_storages = {} # usage => storage_state
-        @_storages_mutex = Mutex.new
       end
 
       def configure(conf)
@@ -95,56 +88,75 @@ module Fluent
           if @_storages[section.usage]
             raise Fluent::ConfigError, "duplicated storages configured: #{section.usage}"
           end
-          config = conf.elements(name: 'storage', arg: section.usage).first
-          raise "storage section with argument '#{section.usage}' not found. it may be a bug." unless config
-
-          storage = Plugin.new_storage(section[:@type])
-          storage.owner = self
-          storage.configure(config)
+          storage = Plugin.new_storage(section[:@type], parent: self)
+          storage.configure(section.corresponding_config_element)
           @_storages[section.usage] = StorageState.new(wrap_instance(storage), false)
         end
       end
 
-      def stop
+      def start
         super
-        # timer stops automatically
+
+        @_storages_started = true
+        @_storages.each_pair do |usage, s|
+          s.storage.start
+          s.storage.load
+
+          if s.storage.autosave && !s.storage.persistent
+            timer_execute(:storage_autosave, s.storage.autosave_interval, repeat: true) do
+              begin
+                s.storage.save
+              rescue => e
+                log.error "plugin storage failed to save its data", usage: usage, type: type, error: e
+              end
+            end
+          end
+          s.running = true
+        end
       end
 
-      def shutdown
+      def storage_operate(method_name, &block)
         @_storages.each_pair do |usage, s|
           begin
-            s.storage.save if s.storage.save_at_shutdown
+            block.call(s) if block_given?
+            s.storage.send(method_name)
           rescue => e
-            log.error "unexpected error while saving data of plugin storages", usage: usage, storage: s.storage, error: e
+            log.error "unexpected error while #{method_name}", usage: usage, storage: s.storage, error: e
           end
         end
+      end
 
+      def stop
         super
+        # timer stops automatically in super
+        storage_operate(:stop)
       end
 
-      def close
-        @_storages.each_pair do |usage, s|
-          begin
-            s.storage.close
-          rescue => e
-            log.error "unexpected error while closing plugin storages", usage: usage, storage: s.storage, error: e
-          end
-          s.running = false
+      def before_shutdown
+        storage_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        storage_operate(:shutdown) do |s|
+          s.storage.save if s.storage.save_at_shutdown
         end
+        super
+      end
+
+      def after_shutdown
+        storage_operate(:after_shutdown)
+        super
+      end
 
+      def close
+        storage_operate(:close){|s| s.running = false }
         super
       end
 
       def terminate
-        @_storages.each_pair do |usage, s|
-          begin
-            s.storage.terminate
-          rescue => e
-            log.error "unexpected error while terminating plugin storages", usage: usage, storage: s.storage, error: e
-          end
-        end
+        storage_operate(:terminate)
         @_storages = {}
-
         super
       end
 
@@ -170,7 +182,8 @@ module Fluent
         end
 
         def_delegators :@storage, :autosave_interval, :save_at_shutdown
-        def_delegators :@storage, :close, :terminate
+        def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate
+        def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?
 
         def persistent_always?
           true
@@ -257,7 +270,8 @@ module Fluent
 
         def_delegators :@storage, :persistent, :autosave, :autosave_interval, :save_at_shutdown
         def_delegators :@storage, :persistent_always?
-        def_delegators :@storage, :close, :terminate
+        def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate
+        def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?
 
         def synchronized?
           true