fluentd 0.14.0 → 0.14.1

data/lib/fluent/compat/set_time_key_mixin.rb

@@ -0,0 +1,69 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+require 'fluent/compat/record_filter_mixin'
+require 'fluent/time'
+require 'fluent/timezone'
+
+module Fluent
+  module Compat
+    module SetTimeKeyMixin
+      include RecordFilterMixin
+
+      attr_accessor :include_time_key, :time_key, :localtime, :timezone
+
+      def configure(conf)
+        @include_time_key = false
+        @localtime = false
+        @timezone = nil
+
+        super
+
+        if s = conf['include_time_key']
+          include_time_key = Fluent::Config.bool_value(s)
+          raise Fluent::ConfigError, "Invalid boolean expression '#{s}' for include_time_key parameter" if include_time_key.nil?
+
+          @include_time_key = include_time_key
+        end
+
+        if @include_time_key
+          @time_key     = conf['time_key'] || 'time'
+          @time_format  = conf['time_format']
+
+          if    conf['localtime']
+            @localtime = true
+          elsif conf['utc']
+            @localtime = false
+          end
+
+          if conf['timezone']
+            @timezone = conf['timezone']
+            Fluent::Timezone.validate!(@timezone)
+          end
+
+          @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)
+        end
+      end
+
+      def filter_record(tag, time, record)
+        super
+
+        record[@time_key] = @timef.format(time) if @include_time_key
+      end
+    end
+  end
+end

data/lib/fluent/compat/type_converter.rb

@@ -0,0 +1,90 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module TypeConverter
+      Converters = {
+        'string' => lambda { |v| v.to_s },
+        'integer' => lambda { |v| v.to_i },
+        'float' => lambda { |v| v.to_f },
+        'bool' => lambda { |v|
+          case v.downcase
+          when 'true', 'yes', '1'
+            true
+          else
+            false
+          end
+        },
+        'time' => lambda { |v, time_parser|
+          time_parser.parse(v)
+        },
+        'array' => lambda { |v, delimiter|
+          v.to_s.split(delimiter)
+        }
+      }
+
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :types, :string, default: nil
+          config_param :types_delimiter, :string, default: ','
+          config_param :types_label_delimiter, :string, default: ':'
+        }
+      end
+
+      def configure(conf)
+        super
+
+        @type_converters = nil
+        @type_converters = parse_types_parameter unless @types.nil?
+      end
+
+      private
+
+      def convert_type(name, value)
+        converter = @type_converters[name]
+        converter.nil? ? value : converter.call(value)
+      end
+
+      def parse_types_parameter
+        converters = {}
+
+        @types.split(@types_delimiter).each { |pattern_name|
+          name, type, format = pattern_name.split(@types_label_delimiter, 3)
+          raise ConfigError, "Type is needed" if type.nil?
+
+          case type
+          when 'time'
+            require 'fluent/parser'
+            t_parser = Fluent::TextParser::TimeParser.new(format)
+            converters[name] = lambda { |v|
+              Converters[type].call(v, t_parser)
+            }
+          when 'array'
+            delimiter = format || ','
+            converters[name] = lambda { |v|
+              Converters[type].call(v, delimiter)
+            }
+          else
+            converters[name] = Converters[type]
+          end
+        }
+
+        converters
+      end
+    end
+  end
+end

data/lib/fluent/config/configure_proxy.rb

@@ -19,7 +19,7 @@ module Fluent
     class ConfigureProxy
       attr_accessor :name, :final, :param_name, :init, :required, :multi, :alias, :configured_in_section
       attr_accessor :argument, :params, :defaults, :descriptions, :sections
-      # config_param :desc, :string, :default => '....'
+      # config_param :desc, :string, default: '....'
       # config_set_default :buffer_type, :memory
       #
       # config_section :default, required: true, multi: false do
@@ -37,10 +37,16 @@ module Fluent
       #   end
       # end
 
-      def initialize(name, param_name: nil, final: nil, init: nil, required: nil, multi: nil, alias: nil, type_lookup:)
+      def initialize(name, root: false, param_name: nil, final: nil, init: nil, required: nil, multi: nil, alias: nil, type_lookup:)
         @name = name.to_sym
         @final = final
 
+        # For ConfigureProxy of root section, "@name" should be a class name of plugins.
+        # Otherwise (like subsections), "@name" should be a name of section, like "buffer", "store".
+        # For subsections, name will be used as parameter names (unless param_name exists), so overriding proxy's name
+        #   should override "@name".
+        @root_section = root
+
         @param_name = param_name && param_name.to_sym
         @init = init
         @required = required
@@ -67,6 +73,10 @@ module Fluent
         @param_name || @name
       end
 
+      def root?
+        @root_section
+      end
+
       def init?
         @init.nil? ? false : @init
       end
@@ -105,7 +115,12 @@ module Fluent
         options[:final] = @final || other.final
         options[:type_lookup] = @type_lookup
 
-        merged = self.class.new(@name, options)
+        merged = if self.root?
+                   options[:root] = true
+                   self.class.new(other.name, options)
+                 else
+                   self.class.new(@name, options)
+                 end
 
         # configured_in MUST be kept
         merged.configured_in_section = self.configured_in_section
@@ -155,7 +170,12 @@ module Fluent
         options[:final]  = true
         options[:type_lookup] = @type_lookup
 
-        merged = self.class.new(@name, options)
+        merged = if self.root?
+                   options[:root] = true
+                   self.class.new(other.name, options)
+                 else
+                   self.class.new(@name, options)
+                 end
 
         merged.configured_in_section = self.configured_in_section
 

data/lib/fluent/config/dsl.rb

@@ -35,14 +35,19 @@ module Fluent
       end
 
       class Proxy
-        def initialize(name, arg)
+        def initialize(name, arg, include_basepath = Dir.pwd)
           @element = Element.new(name, arg, self)
+          @include_basepath = include_basepath
         end
 
         def element
           @element
         end
 
+        def include_basepath
+          @include_basepath
+        end
+
         def eval(source, source_path)
           @element.instance_eval(source, source_path)
           self
@@ -100,6 +105,18 @@ module Fluent
           self
         end
 
+        def include(*args)
+          ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
+          if args.first =~ /\.rb$/
+            path = File.expand_path(args.first)
+            data = File.read(path)
+            self.instance_eval(data, path)
+          else
+            ss = StringScanner.new('')
+            Config::V1Parser.new(ss, @proxy.include_basepath, '', nil).eval_include(@attrs, @elements, args.first)
+          end
+        end
+
         def source(&block)
           @proxy.add_element('source', nil, block)
         end

data/lib/fluent/config/v1_parser.rb

@@ -148,14 +148,15 @@ module Fluent
 
       def eval_include(attrs, elems, uri)
         u = URI.parse(uri)
-        if u.scheme == 'file' || u.path == uri  # file path
+        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri # file path
+          # When the Windows absolute path then u.scheme.length == 1
+          # e.g. C:
           path = u.path
           if path[0] != ?/
             pattern = File.expand_path("#{@include_basepath}/#{path}")
           else
             pattern = path
           end
-
           Dir.glob(pattern).sort.each { |entry|
             basepath = File.dirname(entry)
             fname = File.basename(entry)

data/lib/fluent/configurable.rb

@@ -119,7 +119,7 @@ module Fluent
         map = configure_proxy_map
         unless map[mod_name]
           type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }
-          proxy = Fluent::Config::ConfigureProxy.new(mod_name, required: true, multi: false, type_lookup: type_lookup)
+          proxy = Fluent::Config::ConfigureProxy.new(mod_name, root: true, required: true, multi: false, type_lookup: type_lookup)
           map[mod_name] = proxy
         end
         map[mod_name]

data/lib/fluent/event.rb

@@ -21,11 +21,25 @@ module Fluent
     include Enumerable
     include MessagePackFactory::Mixin
 
+    # dup does deep copy for event stream
+    def dup
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+
     def size
       raise NotImplementedError, "DO NOT USE THIS CLASS directly."
     end
     alias :length :size
 
+    def empty?
+      size == 0
+    end
+
+    # for tests
+    def ==(other)
+      other.is_a?(EventStream) && self.to_msgpack_stream == other.to_msgpack_stream
+    end
+
     def repeatable?
       false
     end
@@ -70,6 +84,14 @@ module Fluent
       true
     end
 
+    def slice(index, num)
+      if index > 0 || num == 0
+        ArrayEventStream.new([])
+      else
+        self.dup
+      end
+    end
+
     def each(&block)
       block.call(@time, @record)
       nil
@@ -86,7 +108,7 @@ module Fluent
     end
 
     def dup
-      entries = @entries.map { |entry| entry.dup } # @entries.map(:dup) doesn't work by ArgumentError
+      entries = @entries.map{ |time, record| [time, record.dup] }
       ArrayEventStream.new(entries)
     end
 
@@ -119,17 +141,13 @@ module Fluent
   #  2. add events
   #     stream[tag].add(time, record)
   class MultiEventStream < EventStream
-    def initialize
-      @time_array = []
-      @record_array = []
+    def initialize(time_array = [], record_array = [])
+      @time_array = time_array
+      @record_array = record_array
     end
 
     def dup
-      es = MultiEventStream.new
-      @time_array.zip(@record_array).each { |time, record|
-        es.add(time, record.dup)
-      }
-      es
+      MultiEventStream.new(@time_array.dup, @record_array.map(&:dup))
     end
 
     def size
@@ -166,6 +184,16 @@ module Fluent
       @size = size
     end
 
+    def empty?
+      # This is not correct, but actual number of records will be shown after iteration, and
+      # "size" argument is always 0 currently (because forward protocol doesn't tell it to destination)
+      false
+    end
+
+    def dup
+      MessagePackEventStream.new(@data.dup, @size)
+    end
+
     def size
       @size
     end

data/lib/fluent/mixin.rb

@@ -14,292 +14,18 @@
 #    limitations under the License.
 #
 
-require 'fluent/timezone'
-require 'fluent/time'
-require 'fluent/config/error'
+require 'fluent/compat/record_filter_mixin'
+require 'fluent/compat/handle_tag_name_mixin'
+require 'fluent/compat/set_time_key_mixin'
+require 'fluent/compat/set_tag_key_mixin'
+require 'fluent/compat/type_converter'
 
-module Fluent
-  class TimeFormatter
-    def initialize(format, localtime, timezone = nil)
-      @tc1 = 0
-      @tc1_str = nil
-      @tc2 = 0
-      @tc2_str = nil
-
-      if format && format =~ /(^|[^%])(%%)*%L|(^|[^%])(%%)*%\d*N/
-        define_singleton_method(:format) {|time|
-          format_with_subsec(time)
-        }
-      else
-        define_singleton_method(:format) {|time|
-          format_without_subsec(time)
-        }
-      end
-
-      if formatter = Fluent::Timezone.formatter(timezone, format)
-        define_singleton_method(:format_nocache) {|time|
-          formatter.call(time)
-        }
-        return
-      end
-
-      if format
-        if localtime
-          define_singleton_method(:format_nocache) {|time|
-            Time.at(time).strftime(format)
-          }
-        else
-          define_singleton_method(:format_nocache) {|time|
-            Time.at(time).utc.strftime(format)
-          }
-        end
-      else
-        if localtime
-          define_singleton_method(:format_nocache) {|time|
-            Time.at(time).iso8601
-          }
-        else
-          define_singleton_method(:format_nocache) {|time|
-            Time.at(time).utc.iso8601
-          }
-        end
-      end
-    end
-
-    def format_without_subsec(time)
-      if @tc1 == time
-        return @tc1_str
-      elsif @tc2 == time
-        return @tc2_str
-      else
-        str = format_nocache(time)
-        if @tc1 < @tc2
-          @tc1 = time
-          @tc1_str = str
-        else
-          @tc2 = time
-          @tc2_str = str
-        end
-        return str
-      end
-    end
-
-    def format_with_subsec(time)
-      if Fluent::EventTime.eq?(@tc1, time)
-        return @tc1_str
-      elsif Fluent::EventTime.eq?(@tc2, time)
-        return @tc2_str
-      else
-        str = format_nocache(time)
-        if @tc1 < @tc2
-          @tc1 = time
-          @tc1_str = str
-        else
-          @tc2 = time
-          @tc2_str = str
-        end
-        return str
-      end
-    end
-
-    def format(time)
-      # will be overridden in initialize
-    end
-
-    def format_nocache(time)
-      # will be overridden in initialize
-    end
-  end
-
-
-  module RecordFilterMixin
-    def filter_record(tag, time, record)
-    end
-
-    def format_stream(tag, es)
-      out = ''
-      es.each {|time,record|
-        tag_temp = tag.dup
-        filter_record(tag_temp, time, record)
-        out << format(tag_temp, time, record)
-      }
-      out
-    end
-  end
-
-  module HandleTagNameMixin
-    include RecordFilterMixin
-
-    attr_accessor :remove_tag_prefix, :remove_tag_suffix, :add_tag_prefix, :add_tag_suffix
-    def configure(conf)
-      super
-
-      @remove_tag_prefix = if conf.has_key?('remove_tag_prefix')
-                             Regexp.new('^' + Regexp.escape(conf['remove_tag_prefix']))
-                           else
-                             nil
-                           end
-
-      @remove_tag_suffix = if conf.has_key?('remove_tag_suffix')
-                             Regexp.new(Regexp.escape(conf['remove_tag_suffix']) + '$')
-                           else
-                             nil
-                           end
-
-      @add_tag_prefix = conf['add_tag_prefix']
-      @add_tag_suffix = conf['add_tag_suffix']
-    end
-
-    def filter_record(tag, time, record)
-      tag.sub!(@remove_tag_prefix, '') if @remove_tag_prefix
-      tag.sub!(@remove_tag_suffix, '') if @remove_tag_suffix
-      tag.insert(0, @add_tag_prefix) if @add_tag_prefix
-      tag << @add_tag_suffix if @add_tag_suffix
-      super(tag, time, record)
-    end
-  end
-
-  module SetTimeKeyMixin
-    require 'fluent/timezone'
-    include RecordFilterMixin
-
-    attr_accessor :include_time_key, :time_key, :localtime, :timezone
-
-    def configure(conf)
-      @include_time_key = false
-      @localtime = false
-      @timezone = nil
-
-      super
-
-      if s = conf['include_time_key']
-        include_time_key = Config.bool_value(s)
-        raise ConfigError, "Invalid boolean expression '#{s}' for include_time_key parameter" if include_time_key.nil?
-
-        @include_time_key = include_time_key
-      end
-
-      if @include_time_key
-        @time_key     = conf['time_key'] || 'time'
-        @time_format  = conf['time_format']
+require 'fluent/time' # Fluent::TimeFormatter
 
-        if    conf['localtime']
-          @localtime = true
-        elsif conf['utc']
-          @localtime = false
-        end
-
-        if conf['timezone']
-          @timezone = conf['timezone']
-          Fluent::Timezone.validate!(@timezone)
-        end
-
-        @timef = TimeFormatter.new(@time_format, @localtime, @timezone)
-      end
-    end
-
-    def filter_record(tag, time, record)
-      super
-
-      record[@time_key] = @timef.format(time) if @include_time_key
-    end
-  end
-
-  module SetTagKeyMixin
-    include RecordFilterMixin
-
-    attr_accessor :include_tag_key, :tag_key
-
-    def configure(conf)
-      @include_tag_key = false
-
-      super
-
-      if s = conf['include_tag_key']
-        include_tag_key = Config.bool_value(s)
-        raise ConfigError, "Invalid boolean expression '#{s}' for include_tag_key parameter" if include_tag_key.nil?
-
-        @include_tag_key = include_tag_key
-      end
-
-      @tag_key = conf['tag_key'] || 'tag' if @include_tag_key
-    end
-
-    def filter_record(tag, time, record)
-      super
-
-      record[@tag_key] = tag if @include_tag_key
-    end
-  end
-
-  module TypeConverter
-    Converters = {
-      'string' => lambda { |v| v.to_s },
-      'integer' => lambda { |v| v.to_i },
-      'float' => lambda { |v| v.to_f },
-      'bool' => lambda { |v|
-        case v.downcase
-        when 'true', 'yes', '1'
-          true
-        else
-          false
-        end
-      },
-      'time' => lambda { |v, time_parser|
-        time_parser.parse(v)
-      },
-      'array' => lambda { |v, delimiter|
-        v.to_s.split(delimiter)
-      }
-    }
-
-    def self.included(klass)
-      klass.instance_eval {
-        config_param :types, :string, default: nil
-        config_param :types_delimiter, :string, default: ','
-        config_param :types_label_delimiter, :string, default: ':'
-      }
-    end
-
-    def configure(conf)
-      super
-
-      @type_converters = nil
-      @type_converters = parse_types_parameter unless @types.nil?
-    end
-
-    private
-
-    def convert_type(name, value)
-      converter = @type_converters[name]
-      converter.nil? ? value : converter.call(value)
-    end
-
-    def parse_types_parameter
-      converters = {}
-
-      @types.split(@types_delimiter).each { |pattern_name|
-        name, type, format = pattern_name.split(@types_label_delimiter, 3)
-        raise ConfigError, "Type is needed" if type.nil?
-
-        case type
-        when 'time'
-          require 'fluent/parser'
-          t_parser = Fluent::TextParser::TimeParser.new(format)
-          converters[name] = lambda { |v|
-            Converters[type].call(v, t_parser)
-          }
-        when 'array'
-          delimiter = format || ','
-          converters[name] = lambda { |v|
-            Converters[type].call(v, delimiter)
-          }
-        else
-          converters[name] = Converters[type]
-        end
-      }
-
-      converters
-    end
-  end
+module Fluent
+  RecordFilterMixin = Fluent::Compat::RecordFilterMixin
+  HandleTagNameMixin = Fluent::Compat::HandleTagNameMixin
+  SetTimeKeyMixin = Fluent::Compat::SetTimeKeyMixin
+  SetTagKeyMixin = Fluent::Compat::SetTagKeyMixin
+  TypeConverter = Fluent::Compat::TypeConverter
 end