fluentd 0.12.40 → 1.6.2

data/lib/fluent/compat/output_chain.rb

@@ -0,0 +1,60 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'singleton'
+
+module Fluent
+  module Compat
+    # TODO: remove when old plugin API are removed
+    class NullOutputChain
+      include Singleton
+
+      def next
+      end
+    end
+
+    class OutputChain
+      def initialize(array, tag, es, chain=NullOutputChain.instance)
+        @array = array
+        @tag = tag
+        @es = es
+        @offset = 0
+        @chain = chain
+      end
+
+      def next
+        if @array.length <= @offset
+          return @chain.next
+        end
+        @offset += 1
+        @array[@offset-1].emit_events(@tag, @es)
+        self.next
+      end
+    end
+
+    class CopyOutputChain < OutputChain
+      def next
+        if @array.length <= @offset
+          return @chain.next
+        end
+        @offset += 1
+        es = @array.length > @offset ? @es.dup : @es
+        @array[@offset-1].emit_events(@tag, es)
+        self.next
+      end
+    end
+  end
+end

data/lib/fluent/compat/parser.rb

@@ -0,0 +1,310 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/parser'
+require 'fluent/mixin'
+
+require 'fluent/config'
+require 'fluent/compat/type_converter'
+
+require 'fluent/plugin/parser_regexp'
+require 'fluent/plugin/parser_json'
+require 'fluent/plugin/parser_tsv'
+require 'fluent/plugin/parser_ltsv'
+require 'fluent/plugin/parser_csv'
+require 'fluent/plugin/parser_none'
+require 'fluent/plugin/parser_apache2'
+require 'fluent/plugin/parser_syslog'
+require 'fluent/plugin/parser_multiline'
+
+module Fluent
+  module Compat
+    class Parser < Fluent::Plugin::Parser
+      # TODO: warn when deprecated
+    end
+
+    class TextParser
+      # Keep backward compatibility for existing plugins
+      ParserError = Fluent::Plugin::Parser::ParserError
+      # TODO: will be removed at v1
+      TypeConverter = Fluent::TypeConverter
+
+      def initialize
+        # TODO: warn when deprecated
+        @parser = nil
+        @estimate_current_event = nil
+      end
+
+      attr_reader :parser
+
+      # SET false BEFORE CONFIGURE, to return nil when time not parsed
+      # 'configure()' may raise errors for unexpected configurations
+      attr_accessor :estimate_current_event
+
+      def configure(conf, required=true)
+        format = conf['format']
+
+        @parser = TextParser.lookup(format)
+
+        if @parser.respond_to?(:configure)
+          @parser.configure(conf)
+        end
+        if !@estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
+          # external code sets parser.estimate_current_event = false
+          @parser.estimate_current_event = @estimate_current_event
+        end
+
+        return true
+      end
+
+      def parse(text, &block)
+        if block
+          @parser.parse(text, &block)
+        else
+          @parser.parse(text) { |time, record|
+            return time, record
+          }
+        end
+      end
+
+      def self.register_template(type, template, time_format=nil)
+        # TODO: warn when deprecated to use Plugin.register_parser directly
+        if template.is_a?(Class) || template.respond_to?(:call)
+          Fluent::Plugin.register_parser(type, template)
+        elsif template.is_a?(Regexp)
+          Fluent::Plugin.register_parser(type, Proc.new { RegexpParser.new(template, {'time_format' => time_format}) })
+        else
+          raise ArgumentError, "Template for parser must be a Class, callable object or regular expression object"
+        end
+      end
+
+      def self.lookup(format)
+        # TODO: warn when deprecated to use Plugin.new_parser or RegexpParser.new directly
+        if format.nil?
+          raise ConfigError, "'format' parameter is required"
+        end
+
+        if format[0] == ?/ && format[format.length-1] == ?/
+          # regexp
+          begin
+            regexp = Regexp.new(format[1..-2])
+            if regexp.named_captures.empty?
+              raise "No named captures"
+            end
+          rescue
+            raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
+          end
+
+          RegexpParser.new(regexp)
+        else
+          # built-in template
+          begin
+            Fluent::Plugin.new_parser(format)
+          rescue ConfigError # keep same error message
+            raise ConfigError, "Unknown format template '#{format}'"
+          end
+        end
+      end
+
+      module TypeConverterCompatParameters
+        def convert_type_converter_parameters!(conf)
+          if conf["types"]
+            delimiter = conf["types_delimiter"] || ','
+            label_delimiter = conf["types_label_delimiter"] || ':'
+            types = {}
+            conf['types'].split(delimiter).each do |pair|
+              key, value = pair.split(label_delimiter, 2)
+              if value.start_with?("time#{label_delimiter}")
+                value = value.split(label_delimiter, 2).join(':')
+              elsif value.start_with?("array#{label_delimiter}")
+                value = value.split(label_delimiter, 2).join(':')
+              end
+              types[key] = value
+            end
+            conf["types"] = JSON.dump(types)
+          end
+        end
+      end
+
+      class TimeParser < Fluent::TimeParser
+        # TODO: warn when deprecated
+      end
+
+      class RegexpParser < Fluent::Plugin::RegexpParser
+        include TypeConverterCompatParameters
+
+        # TODO: warn when deprecated
+        def initialize(regexp, conf = {})
+          super()
+
+          @stored_regexp = regexp
+          @manually_configured = false
+          unless conf.empty?
+            conf_init = if conf.is_a?(Fluent::Config::Element)
+                          conf
+                        else
+                          Fluent::Config::Element.new('parse', '', conf, [])
+                        end
+            self.configure(conf_init)
+            @manually_configured = true
+          end
+        end
+
+        def configure(conf)
+          return if @manually_configured # not to run twice
+
+          conf['expression'] ||= @stored_regexp.source
+          conf['ignorecase'] ||= @stored_regexp.options & Regexp::IGNORECASE != 0
+          conf['multiline'] ||= @stored_regexp.options & Regexp::MULTILINE != 0
+          convert_type_converter_parameters!(conf)
+
+          super
+        end
+
+        def patterns
+          {'format' => @regexp, 'time_format' => @time_format}
+        end
+      end
+
+      class ValuesParser < Parser
+        include Fluent::Compat::TypeConverter
+
+        config_param :keys, :array, default: []
+        config_param :time_key, :string, default: nil
+        config_param :null_value_pattern, :string, default: nil
+        config_param :null_empty_string, :bool, default: false
+
+        def configure(conf)
+          super
+
+          if @time_key && !@keys.include?(@time_key) && @estimate_current_event
+            raise Fluent::ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
+          end
+
+          if @time_format && !@time_key
+            raise Fluent::ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
+          end
+
+          @time_parser = time_parser_create
+
+          if @null_value_pattern
+            @null_value_pattern = Regexp.new(@null_value_pattern)
+          end
+
+          @mutex = Mutex.new
+        end
+
+        def values_map(values)
+          record = Hash[keys.zip(values.map { |value| convert_value_to_nil(value) })]
+
+          if @time_key
+            value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
+            time = if value.nil?
+                     if @estimate_current_event
+                       Fluent::EventTime.now
+                     else
+                       nil
+                     end
+                   else
+                     @mutex.synchronize { @time_parser.parse(value) }
+                   end
+          elsif @estimate_current_event
+            time = Fluent::EventTime.now
+          else
+            time = nil
+          end
+
+          convert_field_type!(record) if @type_converters
+
+          return time, record
+        end
+
+        private
+
+        def convert_field_type!(record)
+          @type_converters.each_key { |key|
+            if value = record[key]
+              record[key] = convert_type(key, value)
+            end
+          }
+        end
+
+        def convert_value_to_nil(value)
+          if value and @null_empty_string
+            value = (value == '') ? nil : value
+          end
+          if value and @null_value_pattern
+            value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
+          end
+          value
+        end
+      end
+
+      class JSONParser < Fluent::Plugin::JSONParser
+        include TypeConverterCompatParameters
+        # TODO: warn when deprecated
+        def configure(conf)
+          convert_type_converter_parameters!(conf)
+          super
+        end
+      end
+
+      class TSVParser < Fluent::Plugin::TSVParser
+        include TypeConverterCompatParameters
+        # TODO: warn when deprecated
+        def configure(conf)
+          convert_type_converter_parameters!(conf)
+          super
+        end
+      end
+
+      class LabeledTSVParser < Fluent::Plugin::LabeledTSVParser
+        include TypeConverterCompatParameters
+        # TODO: warn when deprecated
+        def configure(conf)
+          convert_type_converter_parameters!(conf)
+          super
+        end
+      end
+
+      class CSVParser < Fluent::Plugin::CSVParser
+        include TypeConverterCompatParameters
+        # TODO: warn when deprecated
+        def configure(conf)
+          convert_type_converter_parameters!(conf)
+          super
+        end
+      end
+
+      class NoneParser < Fluent::Plugin::NoneParser
+        # TODO: warn when deprecated
+      end
+
+      class ApacheParser < Fluent::Plugin::Apache2Parser
+        # TODO: warn when deprecated
+      end
+
+      class SyslogParser < Fluent::Plugin::SyslogParser
+        # TODO: warn when deprecated
+      end
+
+      class MultilineParser < Fluent::Plugin::MultilineParser
+        # TODO: warn when deprecated
+      end
+    end
+  end
+end

data/lib/fluent/compat/parser_utils.rb

@@ -0,0 +1,40 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/compat_parameters'
+
+module Fluent
+  module Compat
+    module ParserUtils
+      PARSER_PARAMS = Fluent::PluginHelper::CompatParameters::PARSER_PARAMS
+
+      def self.convert_parser_conf(conf)
+        return if conf.elements(name: 'parse').first
+
+        parser_params = {}
+        PARSER_PARAMS.each do |older, newer|
+          next unless newer
+          if conf.has_key?(older)
+            parser_params[newer] = conf[older]
+          end
+        end
+        unless parser_params.empty?
+          conf.elements << Fluent::Config::Element.new('parse', '', parser_params, [])
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/propagate_default.rb

@@ -0,0 +1,62 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/configurable'
+
+module Fluent
+  module Compat
+    module PropagateDefault
+      # This mixin is to prepend to 3rd party plugins of v0.12 APIs.
+      # 3rd party plugins may override default values of some parameters, like `buffer_type`.
+      # But default values of such parameters will NOT used, but defaults of <buffer>@type</buffer>
+      # will be used in fact. It should bring troubles.
+      # This mixin defines Class method .config_param and .config_set_default (which should be used by extend)
+      # to propagate changes of default values to subsections.
+      def self.included(mod)
+        mod.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        CONFIGURABLE_CLASS_METHODS = Fluent::Configurable::ClassMethods
+
+        def config_param(name, type = nil, **kwargs, &block)
+          CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind(self).call(name, type, **kwargs, &block)
+          pparams = propagate_default_params
+          if kwargs.has_key?(:default) && pparams[name.to_s]
+            newer = pparams[name.to_s].to_sym
+            overridden_default_value = kwargs[:default]
+
+            CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind(self).call(:buffer) do
+              config_set_default newer, overridden_default_value
+            end
+          end
+        end
+
+        def config_set_default(name, defval)
+          CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind(self).call(name, defval)
+          pparams = propagate_default_params
+          if pparams[name.to_s]
+            newer = pparams[name.to_s].to_sym
+
+            CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind(self).call(:buffer) do
+              self.config_set_default newer, defval
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/record_filter_mixin.rb

@@ -0,0 +1,34 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module RecordFilterMixin
+      def filter_record(tag, time, record)
+      end
+
+      def format_stream(tag, es)
+        out = ''
+        es.each {|time,record|
+          tag_temp = tag.dup
+          filter_record(tag_temp, time, record)
+          out << format(tag_temp, time, record)
+        }
+        out
+      end
+    end
+  end
+end

data/lib/fluent/compat/set_tag_key_mixin.rb

@@ -0,0 +1,50 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+require 'fluent/config/types'
+require 'fluent/compat/record_filter_mixin'
+
+module Fluent
+  module Compat
+    module SetTagKeyMixin
+      include RecordFilterMixin
+
+      attr_accessor :include_tag_key, :tag_key
+
+      def configure(conf)
+        @include_tag_key = false
+
+        super
+
+        if s = conf['include_tag_key']
+          include_tag_key = Fluent::Config.bool_value(s)
+          raise Fluent::ConfigError, "Invalid boolean expression '#{s}' for include_tag_key parameter" if include_tag_key.nil?
+
+          @include_tag_key = include_tag_key
+        end
+
+        @tag_key = conf['tag_key'] || 'tag' if @include_tag_key
+      end
+
+      def filter_record(tag, time, record)
+        super
+
+        record[@tag_key] = tag if @include_tag_key
+      end
+    end
+  end
+end

data/lib/fluent/compat/set_time_key_mixin.rb

@@ -0,0 +1,69 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+require 'fluent/compat/record_filter_mixin'
+require 'fluent/time'
+require 'fluent/timezone'
+
+module Fluent
+  module Compat
+    module SetTimeKeyMixin
+      include RecordFilterMixin
+
+      attr_accessor :include_time_key, :time_key, :localtime, :timezone
+
+      def configure(conf)
+        @include_time_key = false
+        @localtime = false
+        @timezone = nil
+
+        super
+
+        if s = conf['include_time_key']
+          include_time_key = Fluent::Config.bool_value(s)
+          raise Fluent::ConfigError, "Invalid boolean expression '#{s}' for include_time_key parameter" if include_time_key.nil?
+
+          @include_time_key = include_time_key
+        end
+
+        if @include_time_key
+          @time_key     = conf['time_key'] || 'time'
+          @time_format  = conf['time_format']
+
+          if    conf['localtime']
+            @localtime = true
+          elsif conf['utc']
+            @localtime = false
+          end
+
+          if conf['timezone']
+            @timezone = conf['timezone']
+            Fluent::Timezone.validate!(@timezone)
+          end
+
+          @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)
+        end
+      end
+
+      def filter_record(tag, time, record)
+        super
+
+        record[@time_key] = @timef.format(time) if @include_time_key
+      end
+    end
+  end
+end