fluentd 0.12.40 → 0.14.0

data/lib/fluent/compat/output_chain.rb

@@ -0,0 +1,60 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'singleton'
+
+module Fluent
+  module Compat
+    # TODO: remove when old plugin API are removed
+    class NullOutputChain
+      include Singleton
+
+      def next
+      end
+    end
+
+    class OutputChain
+      def initialize(array, tag, es, chain=NullOutputChain.instance)
+        @array = array
+        @tag = tag
+        @es = es
+        @offset = 0
+        @chain = chain
+      end
+
+      def next
+        if @array.length <= @offset
+          return @chain.next
+        end
+        @offset += 1
+        @array[@offset-1].emit_events(@tag, @es)
+        self.next
+      end
+    end
+
+    class CopyOutputChain < OutputChain
+      def next
+        if @array.length <= @offset
+          return @chain.next
+        end
+        @offset += 1
+        es = @array.length > @offset ? @es.dup : @es
+        @array[@offset-1].emit_events(@tag, es)
+        self.next
+      end
+    end
+  end
+end

data/lib/fluent/compat/parser.rb

@@ -0,0 +1,163 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/parser'
+require 'fluent/mixin'
+
+require 'fluent/plugin/parser_json'
+require 'fluent/plugin/parser_tsv'
+require 'fluent/plugin/parser_ltsv'
+require 'fluent/plugin/parser_csv'
+require 'fluent/plugin/parser_none'
+require 'fluent/plugin/parser_apache2'
+require 'fluent/plugin/parser_syslog'
+require 'fluent/plugin/parser_multiline'
+
+module Fluent
+  module Compat
+    class Parser < Fluent::Plugin::Parser
+      # TODO: warn when deprecated
+    end
+
+    class TextParser
+      # Keep backward compatibility for existing plugins
+      ParserError = Fluent::Plugin::Parser::ParserError
+      # TODO: will be removed at v1
+      TypeConverter = Fluent::TypeConverter
+
+      def initialize
+        # TODO: warn when deprecated
+        @parser = nil
+        @estimate_current_event = nil
+      end
+
+      attr_reader :parser
+
+      # SET false BEFORE CONFIGURE, to return nil when time not parsed
+      # 'configure()' may raise errors for unexpected configurations
+      attr_accessor :estimate_current_event
+
+      def configure(conf, required=true)
+        format = conf['format']
+
+        @parser = TextParser.lookup(format)
+        if ! @estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
+          @parser.estimate_current_event = @estimate_current_event
+        end
+
+        if @parser.respond_to?(:configure)
+          @parser.configure(conf)
+        end
+
+        return true
+      end
+
+      def parse(text, &block)
+        if block
+          @parser.parse(text, &block)
+        else
+          @parser.parse(text) { |time, record|
+            return time, record
+          }
+        end
+      end
+
+      def self.register_template(type, template, time_format=nil)
+        # TODO: warn when deprecated to use Plugin.register_parser directly
+        if template.is_a?(Class) || template.respond_to?(:call)
+          Fluent::Plugin.register_parser(type, template)
+        elsif template.is_a?(Regexp)
+          Fluent::Plugin.register_parser(type, Proc.new { RegexpParser.new(template, {'time_format' => time_format}) })
+        else
+          raise ArgumentError, "Template for parser must be a Class, callable object or regular expression object"
+        end
+      end
+
+      def self.lookup(format)
+        # TODO: warn when deprecated to use Plugin.new_parser or RegexpParser.new directly
+        if format.nil?
+          raise ConfigError, "'format' parameter is required"
+        end
+
+        if format[0] == ?/ && format[format.length-1] == ?/
+          # regexp
+          begin
+            regexp = Regexp.new(format[1..-2])
+            if regexp.named_captures.empty?
+              raise "No named captures"
+            end
+          rescue
+            raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
+          end
+
+          RegexpParser.new(regexp)
+        else
+          # built-in template
+          begin
+            Fluent::Plugin.new_parser(format)
+          rescue ConfigError # keep same error message
+            raise ConfigError, "Unknown format template '#{format}'"
+          end
+        end
+      end
+
+      class TimeParser < Fluent::Plugin::Parser::TimeParser
+        # TODO: warn when deprecated
+      end
+
+      class RegexpParser < Fluent::Plugin::RegexpParser
+        # TODO: warn when deprecated
+      end
+
+      class ValuesParser < Fluent::Plugin::ValuesParser
+        # TODO: warn when deprecated
+      end
+
+      class JSONParser < Fluent::Plugin::JSONParser
+        # TODO: warn when deprecated
+      end
+
+      class TSVParser < Fluent::Plugin::TSVParser
+        # TODO: warn when deprecated
+      end
+
+      class LabeledTSVParser < Fluent::Plugin::LabeledTSVParser
+        # TODO: warn when deprecated
+      end
+
+      class CSVParser < Fluent::Plugin::CSVParser
+        # TODO: warn when deprecated
+      end
+
+      class NoneParser < Fluent::Plugin::NoneParser
+        # TODO: warn when deprecated
+      end
+
+      class ApacheParser < Fluent::Plugin::Apache2Parser
+        # TODO: warn when deprecated
+      end
+
+      class SyslogParser < Fluent::Plugin::SyslogParser
+        # TODO: warn when deprecated
+      end
+
+      class MultilineParser < Fluent::Plugin::MultilineParser
+        # TODO: warn when deprecated
+      end
+    end
+  end
+end

data/lib/fluent/compat/propagate_default.rb

@@ -0,0 +1,62 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/configurable'
+
+module Fluent
+  module Compat
+    module PropagateDefault
+      # This mixin is to prepend to 3rd party plugins of v0.12 APIs.
+      # 3rd party plugins may override default values of some parameters, like `buffer_type`.
+      # But default values of such parameters will NOT used, but defaults of <buffer>@type</buffer>
+      # will be used in fact. It should bring troubles.
+      # This mixin defines Class method .config_param and .config_set_default (which should be used by extend)
+      # to propagate changes of default values to subsections.
+      def self.included(mod)
+        mod.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        CONFIGURABLE_CLASS_METHODS = Fluent::Configurable::ClassMethods
+
+        def config_param(name, type = nil, **kwargs, &block)
+          CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind(self).call(name, type, **kwargs, &block)
+          pparams = propagate_default_params
+          if kwargs.has_key?(:default) && pparams[name.to_s]
+            newer = pparams[name.to_s].to_sym
+            overridden_default_value = kwargs[:default]
+
+            CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind(self).call(:buffer) do
+              config_set_default newer, overridden_default_value
+            end
+          end
+        end
+
+        def config_set_default(name, defval)
+          CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind(self).call(name, defval)
+          pparams = propagate_default_params
+          if pparams[name.to_s]
+            newer = pparams[name.to_s].to_sym
+
+            CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind(self).call(:buffer) do
+              self.config_set_default newer, defval
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config.rb

@@ -20,18 +20,32 @@ require 'fluent/configurable'
 
 module Fluent
   module Config
-    def self.parse(str, fname, basepath = Dir.pwd, v1_config = false)
-      if fname =~ /\.rb$/
+    def self.parse(str, fname, basepath = Dir.pwd, v1_config = nil, syntax: :v1)
+      parser = if fname =~ /\.rb$/ || syntax == :ruby
+                 :ruby
+               elsif v1_config.nil?
+                 case syntax
+                 when :v1 then :v1
+                 when :v0 then :v0
+                 else
+                   raise ArgumentError, "Unknown Fluentd configuration syntax: '#{syntax}'"
+                 end
+               elsif v1_config then :v1
+               else :v0
+               end
+      case parser
+      when :v1
+        require 'fluent/config/v1_parser'
+        V1Parser.parse(str, fname, basepath, Kernel.binding)
+      when :v0
+        # TODO: show deprecated message in v1
+        require 'fluent/config/parser'
+        Parser.parse(str, fname, basepath)
+      when :ruby
         require 'fluent/config/dsl'
         Config::DSL::Parser.parse(str, File.join(basepath, fname))
       else
-        if v1_config
-          require 'fluent/config/v1_parser'
-          V1Parser.parse(str, fname, basepath, Kernel.binding)
-        else
-          require 'fluent/config/parser'
-          Parser.parse(str, fname, basepath)
-        end
+        raise "[BUG] unknown configuration parser specification:'#{parser}'"
       end
     end
 
@@ -39,15 +53,4 @@ module Fluent
       Element.new(name, '', {}, [])
     end
   end
-
-  module PluginId
-    def configure(conf)
-      @id = conf['@id'] || conf['id']
-      super
-    end
-
-    def plugin_id
-      @id ? @id : "object:#{object_id.to_s(16)}"
-    end
-  end
 end

data/lib/fluent/config/configure_proxy.rb

@@ -14,12 +14,11 @@
 #    limitations under the License.
 #
 
-require 'fluent/configurable'
-
 module Fluent
   module Config
     class ConfigureProxy
-      attr_accessor :name, :final, :param_name, :required, :multi, :alias, :argument, :params, :defaults, :descriptions, :sections
+      attr_accessor :name, :final, :param_name, :init, :required, :multi, :alias, :configured_in_section
+      attr_accessor :argument, :params, :defaults, :descriptions, :sections
       # config_param :desc, :string, :default => '....'
       # config_set_default :buffer_type, :memory
       #
@@ -38,20 +37,38 @@ module Fluent
       #   end
       # end
 
-      def initialize(name, opts = {})
+      def initialize(name, param_name: nil, final: nil, init: nil, required: nil, multi: nil, alias: nil, type_lookup:)
         @name = name.to_sym
-        @final = opts.fetch(:final, false)
+        @final = final
+
+        @param_name = param_name && param_name.to_sym
+        @init = init
+        @required = required
+        @multi = multi
+        @alias = binding.local_variable_get(:alias)
+        @type_lookup = type_lookup
+
+        raise "init and required are exclusive" if @init && @required
 
-        @param_name = (opts[:param_name] || @name).to_sym
-        @required = opts[:required]
-        @multi = opts[:multi]
-        @alias = opts[:alias]
+        # specify section name for viewpoint of owner(parent) plugin
+        # for buffer plugins: all params are in <buffer> section of owner
+        # others: <storage>, <format> (formatter/parser), ...
+        @configured_in_section = nil
 
         @argument = nil # nil: ignore argument
         @params = {}
         @defaults = {}
         @descriptions = {}
         @sections = {}
+        @current_description = nil
+      end
+
+      def variable_name
+        @param_name || @name
+      end
+
+      def init?
+        @init.nil? ? false : @init
       end
 
       def required?
@@ -63,21 +80,38 @@ module Fluent
       end
 
       def final?
-        @final
+        !!@final
       end
 
       def merge(other) # self is base class, other is subclass
         return merge_for_finalized(other) if self.final?
 
-        options = {
-          param_name: other.param_name,
-          required: (other.required.nil? ? self.required : other.required),
-          multi: (other.multi.nil? ? self.multi : other.multi)
-        }
-        merged = self.class.new(other.name, options)
+        [:param_name, :required, :multi, :alias, :configured_in_section].each do |prohibited_name|
+          if overwrite?(other, prohibited_name)
+            raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: #{prohibited_name}"
+          end
+        end
+
+        options = {}
+        # param_name affects instance variable name, which is just "internal" of each plugins.
+        # so it must not be changed. base class's name (or param_name) is always used.
+        options[:param_name] = @param_name
+
+        # subclass cannot overwrite base class's definition
+        options[:init] = @init.nil? ? other.init : self.init
+        options[:required] = @required.nil? ? other.required : self.required
+        options[:multi] = @multi.nil? ? other.multi : self.multi
+        options[:alias] = @alias.nil? ? other.alias : self.alias
+        options[:final] = @final || other.final
+        options[:type_lookup] = @type_lookup
+
+        merged = self.class.new(@name, options)
+
+        # configured_in MUST be kept
+        merged.configured_in_section = self.configured_in_section
 
         merged.argument = other.argument || self.argument
-        merged.params = self.params.merge(other.params)
+        merged.params = other.params.merge(self.params)
         merged.defaults = self.defaults.merge(other.defaults)
         merged.sections = {}
         (self.sections.keys + other.sections.keys).uniq.each do |section_key|
@@ -98,20 +132,36 @@ module Fluent
 
       def merge_for_finalized(other)
         # list what subclass can do for finalized section
-        #  * overwrite param_name to escape duplicated name of instance variable
         #  * append params/defaults/sections which are missing in superclass
+        #  * change default values of superclass
+        #  * overwrite init to make it enable to instantiate section objects with added default values
+
+        if other.final == false && overwrite?(other, :final)
+          raise ConfigError, "BUG: subclass cannot overwrite finalized base class's config_section"
+        end
+
+        [:param_name, :required, :multi, :alias, :configured_in_section].each do |prohibited_name|
+          if overwrite?(other, prohibited_name)
+            raise ConfigError, "BUG: subclass cannot overwrite base class's config_section: #{prohibited_name}"
+          end
+        end
+
+        options = {}
+        options[:param_name] = @param_name
+        options[:init] = @init || other.init
+        options[:required] = @required.nil? ? other.required : self.required
+        options[:multi] = @multi.nil? ? other.multi : self.multi
+        options[:alias] = @alias.nil? ? other.alias : self.alias
+        options[:final]  = true
+        options[:type_lookup] = @type_lookup
+
+        merged = self.class.new(@name, options)
 
-        options = {
-          param_name: other.param_name,
-          required: (self.required.nil? ? other.required : self.required),
-          multi: (self.multi.nil? ? other.multi : self.multi),
-          final: true,
-        }
-        merged = self.class.new(other.name, options)
+        merged.configured_in_section = self.configured_in_section
 
         merged.argument = self.argument || other.argument
         merged.params = other.params.merge(self.params)
-        merged.defaults = other.defaults.merge(self.defaults)
+        merged.defaults = self.defaults.merge(other.defaults)
         merged.sections = {}
         (self.sections.keys + other.sections.keys).uniq.each do |section_key|
           self_section = self.sections[section_key]
@@ -129,49 +179,33 @@ module Fluent
         merged
       end
 
-      def option_value_type!(name, opts, key, klass)
-        if opts.has_key?(key) && !opts[key].is_a?(klass)
-          raise ArgumentError, "#{name}: #{key} must be a #{klass}, but #{opts[key].class}"
+      def overwrite_defaults(other) # other is owner plugin's corresponding proxy
+        self.defaults = self.defaults.merge(other.defaults)
+        self.sections.keys.each do |section_key|
+          if other.sections.has_key?(section_key)
+            self.sections[section_key].overwrite_defaults(other.sections[section_key])
+          end
         end
       end
 
-      def parameter_configuration(name, *args, &block)
+      def parameter_configuration(name, type = nil, **kwargs, &block)
         name = name.to_sym
 
         opts = {}
-        args.each { |a|
-          if a.is_a?(Symbol)
-            opts[:type] = a
-          elsif a.is_a?(Hash)
-            opts.merge!(a)
-          else
-            raise ArgumentError, "#{self.name}: wrong number of arguments (#{1 + args.length} for #{block ? 2 : 3})"
-          end
-        }
+        opts[:type] = type
+        opts.merge!(kwargs)
 
-        type = opts[:type]
         if block && type
-          raise ArgumentError, "#{name}: both of block and type cannot be specified"
+          raise ArgumentError, "#{self.name}: both of block and type cannot be specified"
         end
 
         begin
           type = :string if type.nil?
-          block ||= Configurable.lookup_type(type)
+          block ||= @type_lookup.call(type)
         rescue ConfigError
           # override error message
-          raise ArgumentError, "#{name}: unknown config_argument type `#{type}'"
-        end
-
-        option_value_type!(name, opts, :desc, String)
-        option_value_type!(name, opts, :alias, Symbol)
-        option_value_type!(name, opts, :deprecated, String)
-        option_value_type!(name, opts, :obsoleted, String)
-        if type == :enum
-          if !opts.has_key?(:list) || !opts[:list].all?{|v| v.is_a?(Symbol) }
-            raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
-          end
+          raise ArgumentError, "#{self.name}: unknown config_argument type `#{type}'"
         end
-        option_value_type!(name, opts, :value_type, Symbol) # hash, array
 
         if opts.has_key?(:default)
           config_set_default(name, opts[:default])
@@ -181,25 +215,28 @@ module Fluent
           config_set_desc(name, opts[:desc])
         end
 
-        if opts[:deprecated] && opts[:obsoleted]
-          raise ArgumentError, "#{name}: both of deprecated and obsoleted cannot be specified at once"
-        end
-
         [name, block, opts]
       end
 
-      def config_argument(name, *args, &block)
+      def configured_in(section_name)
+        if @configured_in_section
+          raise ArgumentError, "#{self.name}: configured_in called twice"
+        end
+        @configured_in_section = section_name.to_sym
+      end
+
+      def config_argument(name, type = nil, **kwargs, &block)
         if @argument
           raise ArgumentError, "#{self.name}: config_argument called twice"
         end
-        name, block, opts = parameter_configuration(name, *args, &block)
+        name, block, opts = parameter_configuration(name, type, **kwargs, &block)
 
         @argument = [name, block, opts]
         name
       end
 
-      def config_param(name, *args, &block)
-        name, block, opts = parameter_configuration(name, *args, &block)
+      def config_param(name, type = nil, **kwargs, &block)
+        name, block, opts = parameter_configuration(name, type, **kwargs, &block)
 
         if @current_description
           config_set_desc(name, @current_description)
@@ -237,20 +274,24 @@ module Fluent
         @current_description = description
       end
 
-      def config_section(name, *args, &block)
+      def config_section(name, **kwargs, &block)
         unless block_given?
-          raise ArgumentError, "#{name}: config_section requires block parameter"
+          raise ArgumentError, "#{self.name}: config_section requires block parameter"
         end
         name = name.to_sym
 
-        opts = {}
-        unless args.empty? || args.size == 1 && args.first.is_a?(Hash)
-          raise ArgumentError, "#{self.name}: unknown config_section arguments: #{args.inspect}"
-        end
-
-        sub_proxy = ConfigureProxy.new(name, *args)
+        sub_proxy = ConfigureProxy.new(name, type_lookup: @type_lookup, **kwargs)
         sub_proxy.instance_exec(&block)
 
+        if sub_proxy.init?
+          if sub_proxy.argument && !sub_proxy.defaults.has_key?(sub_proxy.argument.first)
+            raise ArgumentError, "#{self.name}: init is specified, but default value of argument is missing"
+          end
+          if sub_proxy.params.keys.any?{|param_name| !sub_proxy.defaults.has_key?(param_name)}
+            raise ArgumentError, "#{self.name}: init is specified, but there're parameters without default values"
+          end
+        end
+
         @params.delete(name)
         @sections[name] = sub_proxy
 
@@ -270,6 +311,14 @@ module Fluent
         end
         dumped_config
       end
+
+      private
+
+      def overwrite?(other, attribute_name)
+        value = instance_variable_get("@#{attribute_name}")
+        other_value = other.__send__(attribute_name)
+        !value.nil? && !other_value.nil? && value != other_value
+      end
     end
   end
 end