fluentd 0.12.0.pre.3 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c5a6a2cf3f13289f87a554bfafb531593977ffcd
-  data.tar.gz: bbb612c523e612b177980433bcc86cb7a6571d90
+  metadata.gz: 76237f968249345377ad1587cc876fe7783d5f74
+  data.tar.gz: a61a06b92b56a1ab55c729d1e3500a8856d7aeef
 SHA512:
-  metadata.gz: ea0f0eeb1718e00caef76ae713101292daa2019cab3af19a2e26c4f2d2c74920026757e729b5e57b8f60f98f366035211e206a8f9d97849c3f0bfc0fad8ccdc5
-  data.tar.gz: b44314b0bcd226f4666b1079b297ad6f67a646b6be0f22a4e855e5176caac17cdf7b792a2b760bd0ba05184146cacdacc121f643661f41c41e25ee7d9632b5e3
+  metadata.gz: 145f3e187ed6e48d46ffc08d5c6800c66194b1c726f6d8dea36504856890dc13adde304c7fab420a8055f86a740dd58832a4269cebbf03125e02bbbb2c9e45d4
+  data.tar.gz: a050b3b03cf3d1395caab0f8ea36098edec10aafb826812a40e9494cc645d51bf97d916f70e71bec848389f44d64f7f828f81aeb42e06894878a36b872d6504c

data/fluent.conf

@@ -1,26 +1,31 @@
+# In v1 configuration, type and id are @ prefix parameters.
+# @type and @id are recommended. type and id are still available for backward compatibility
 
 ## built-in TCP input
 ## $ echo <json> | fluent-cat <tag>
 <source>
-  type forward
+  @type forward
+  @id forward_input
 </source>
 
 ## built-in UNIX socket input
 #<source>
-#  type unix
+#  @type unix
 #</source>
 
 # HTTP input
 # http://localhost:8888/<tag>?json=<json>
 <source>
-  type http
+  @type http
+  @id http_input
+
   port 8888
 </source>
 
 ## File input
 ## read apache logs with tag=apache.access
 #<source>
-#  type tail
+#  @type tail
 #  format apache
 #  path /var/log/httpd-access.log
 #  tag apache.access
@@ -31,32 +36,38 @@
 # http://localhost:24220/api/plugins?type=TYPE
 # http://localhost:24220/api/plugins?tag=MYTAG
 <source>
-  type monitor_agent
+  @type monitor_agent
+  @id monitor_agent_input
+
   port 24220
 </source>
 
 # Listen DRb for debug
 <source>
-  type debug_agent
+  @type debug_agent
+  @id debug_agent_input
+
   bind 127.0.0.1
   port 24230
 </source>
 
-
 ## match tag=apache.access and write to file
 #<match apache.access>
-#  type file
+#  @type file
 #  path /var/log/fluent/access
 #</match>
 
 ## match tag=debug.** and dump to console
 <match debug.**>
-  type stdout
+  @type stdout
+  @id stdout_output
 </match>
 
 # match tag=system.** and forward to another fluent server
 <match system.**>
-  type forward
+  @type forward
+  @id forward_output
+
   <server>
     host 192.168.0.11
   </server>
@@ -69,9 +80,9 @@
 
 ## match tag=myapp.** and forward and write to file
 #<match myapp.**>
-#  type copy
+#  @type copy
 #  <store>
-#    type forward
+#    @type forward
 #    buffer_type file
 #    buffer_path /var/log/fluent/myapp-forward
 #    retry_limit 50
@@ -81,20 +92,19 @@
 #    </server>
 #  </store>
 #  <store>
-#    type file
+#    @type file
 #    path /var/log/fluent/myapp
 #  </store>
 #</match>
 
 ## match fluent's internal events
 #<match fluent.**>
-#  type null
+#  @type null
 #</match>
 
 ## match not matched logs and write to file
 #<match **>
-#  type file
+#  @type file
 #  path /var/log/fluent/else
 #  compress gz
 #</match>
-

data/lib/fluent/agent.rb

@@ -27,7 +27,6 @@ module Fluent
     include Configurable
 
     def initialize(opts = {})
-      # initialize Configurable
       super()
 
       @context = nil
@@ -43,6 +42,7 @@ module Fluent
 
     attr_reader :log
     attr_reader :outputs
+    attr_reader :filters
     attr_reader :context
     attr_reader :event_router
     attr_reader :error_collector
@@ -53,7 +53,7 @@ module Fluent
       # initialize <match> and <filter> elements
       conf.elements.select { |e| e.name == 'filter' || e.name == 'match' }.each { |e|
         pattern = e.arg.empty? ? '**' : e.arg
-        type = e['type']
+        type = e['@type'] || e['type']
         if e.name == 'filter'
           add_filter(type, pattern, e)
         else
@@ -62,7 +62,6 @@ module Fluent
       }
     end
 
-    # agent API called by Worker
     def start
       @outputs.each { |o|
         o.start
@@ -75,7 +74,6 @@ module Fluent
       }
     end
 
-    # agent API called by Worker
     def shutdown
       @started_filters.map { |f|
         Thread.new do
@@ -145,7 +143,11 @@ module Fluent
       filter
     end
 
-    def handle_emits_error(tag, es, e)
+    # For handling invalid record
+    def emit_error_event(tag, time, record, error)
+    end
+
+    def handle_emits_error(tag, es, error)
     end
 
     class NoMatchMatch

data/lib/fluent/config.rb

@@ -43,17 +43,10 @@ module Fluent
 
   module PluginId
     def configure(conf)
-      @id = conf['id']
+      @id = conf['@id'] || conf['id']
       super
     end
 
-    def require_id
-      unless @id
-        raise ConfigError, "'id' parameter is required"
-      end
-      @id
-    end
-
     def plugin_id
       @id ? @id : "object:#{object_id.to_s(16)}"
     end

data/lib/fluent/config/v1_parser.rb

@@ -51,6 +51,7 @@ module Fluent
       end
 
       ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
+      RESERVED_PARAMS = %W(@type @id @label)
 
       def parse_element(root_element, elem_name, attrs = {}, elems = [])
         while true
@@ -108,7 +109,7 @@ module Fluent
             else
               if k == '@include'
                 parse_include(attrs, elems)
-              elsif k == '@label'
+              elsif RESERVED_PARAMS.include?(k)
                 v = parse_literal
                 unless line_end
                   parse_error! "expected end of line"

data/lib/fluent/event_router.rb

@@ -42,17 +42,10 @@ module Fluent
       @match_rules = []
       @match_cache = MatchCache.new
       @default_collector = default_collector
-      @emit_error_handler = emit_error_handler || RaiseEmitErrorHandler.new
+      @emit_error_handler = emit_error_handler
       @chain = NullOutputChain.instance
     end
 
-    # Agent implements EmitErrorHandler. See 'fluentd/agent.rb'
-    class RaiseEmitErrorHandler
-      def handle_emits_error(tag, es, error)
-        raise error
-      end
-    end
-
     attr_accessor :default_collector
     attr_accessor :emit_error_handler
 

data/lib/fluent/filter.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   class Filter
     include Configurable

data/lib/fluent/formatter.rb

@@ -17,6 +17,18 @@
 module Fluent
   require 'fluent/registry'
 
+  class Formatter
+    include Configurable
+
+    def configure(conf)
+      super
+    end
+
+    def format(tag, time, record)
+      raise NotImplementedError, "Implement this method in child class"
+    end
+  end
+
   module TextFormatter
     module HandleTagAndTimeMixin
       def self.included(klass)
@@ -50,8 +62,7 @@ module Fluent
       end
     end
 
-    class OutFileFormatter
-      include Configurable
+    class OutFileFormatter < Formatter
       include HandleTagAndTimeMixin
 
       config_param :output_time, :bool, :default => true
@@ -64,10 +75,6 @@ module Fluent
         end
       end
 
-      def configure(conf)
-        super
-      end
-
       def format(tag, time, record)
         filter_record(tag, time, record)
         header = ''
@@ -104,8 +111,7 @@ module Fluent
       end
     end
 
-    class JSONFormatter
-      include Configurable
+    class JSONFormatter < Formatter
       include HandleTagAndTimeMixin
       include StructuredFormatMixin
 
@@ -114,8 +120,7 @@ module Fluent
       end
     end
 
-    class MessagePackFormatter
-      include Configurable
+    class MessagePackFormatter < Formatter
       include HandleTagAndTimeMixin
       include StructuredFormatMixin
 
@@ -124,8 +129,7 @@ module Fluent
       end
     end
 
-    class LabeledTSVFormatter
-      include Configurable
+    class LabeledTSVFormatter < Formatter
       include HandleTagAndTimeMixin
 
       config_param :delimiter, :string, :default => "\t"
@@ -142,8 +146,7 @@ module Fluent
       end
     end
 
-    class CsvFormatter
-      include Configurable
+    class CsvFormatter < Formatter
       include HandleTagAndTimeMixin
 
       config_param :delimiter, :default => ',' do |val|
@@ -173,9 +176,7 @@ module Fluent
       end
     end
 
-    class SingleValueFormatter
-      include Configurable
-
+    class SingleValueFormatter < Formatter
       config_param :message_key, :string, :default => 'message'
       config_param :add_newline, :bool, :default => true
 
@@ -186,7 +187,7 @@ module Fluent
       end
     end
 
-    class ProcWrappedFormatter
+    class ProcWrappedFormatter < Formatter
       def initialize(proc)
         @proc = proc
       end

data/lib/fluent/label.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   require 'fluent/agent'
 

data/lib/fluent/output.rb

@@ -208,7 +208,7 @@ module Fluent
       }
 
       if sconf = conf.elements.select {|e| e.name == 'secondary' }.first
-        type = sconf['type'] || conf['type']
+        type = sconf['@type'] || conf['@type'] || sconf['type'] || conf['type']
         @secondary = Plugin.new_output(type)
         @secondary.configure(sconf)
 
@@ -401,7 +401,7 @@ module Fluent
                # secondary retry
                @retry_wait * (2 ** (@num_errors - 2 - @retry_limit))
              end
-      retry_wait = wait + (rand * (wait / 4.0) - (wait / 8.0))
+      retry_wait = wait.finite? ? wait + (rand * (wait / 4.0) - (wait / 8.0)) : wait
       @max_retry_wait ? [retry_wait, @max_retry_wait].min : retry_wait
     end
 

data/lib/fluent/parser.rb

@@ -17,10 +17,39 @@
 module Fluent
   require 'fluent/registry'
 
-  class TextParser
-    class ParserError < StandardError
+  class ParserError < StandardError
+  end
+
+  class Parser
+    include Configurable
+
+    # SET false BEFORE CONFIGURE, to return nil when time not parsed
+    # 'configure()' may raise errors for unexpected configurations
+    attr_accessor :estimate_current_event
+
+    def initialize
+      super
+      @estimate_current_event = true
     end
 
+    def configure(conf)
+      super
+    end
+
+    def parse(text)
+      raise NotImplementedError, "Implement this method in child class"
+    end
+
+    # Keep backward compatibility for existing plugins
+    def call(*a, &b)
+      parse(*a, &b)
+    end
+  end
+
+  class TextParser
+    # Keep backward compatibility for existing plugins
+    ParserError = ::Fluent::ParserError
+
     class TimeParser
       def initialize(time_format)
         @cache1_key = nil
@@ -128,16 +157,11 @@ module Fluent
       end
     end
 
-    class RegexpParser
-      include Configurable
+    class RegexpParser < Parser
       include TypeConverter
 
       config_param :time_format, :string, :default => nil
 
-      # SET false BEFORE CONFIGURE, to return nil when time not parsed
-      # 'configure()' may raise errors for unexpected configurations
-      attr_accessor :estimate_current_event
-
       def initialize(regexp, conf={})
         super()
         @regexp = regexp
@@ -146,7 +170,6 @@ module Fluent
         end
 
         @time_parser = TimeParser.new(@time_format)
-        @estimate_current_event = true
         @mutex = Mutex.new
       end
 
@@ -159,7 +182,7 @@ module Fluent
         {'format' => @regexp, 'time_format' => @time_format}
       end
 
-      def call(text)
+      def parse(text)
         m = @regexp.match(text)
         unless m
           if block_given?
@@ -200,21 +223,10 @@ module Fluent
       end
     end
 
-    class JSONParser
-      include Configurable
-
+    class JSONParser < Parser
       config_param :time_key, :string, :default => 'time'
       config_param :time_format, :string, :default => nil
 
-      # SET false BEFORE CONFIGURE, to return nil when time not parsed
-      # 'configure()' may raise errors for unexpected configurations
-      attr_accessor :estimate_current_event
-
-      def initialize
-        super
-        @estimate_current_event = true
-      end
-
       def configure(conf)
         super
 
@@ -224,7 +236,7 @@ module Fluent
         end
       end
 
-      def call(text)
+      def parse(text)
         record = Yajl.load(text)
 
         if value = record.delete(@time_key)
@@ -259,21 +271,13 @@ module Fluent
       end
     end
 
-    class ValuesParser
-      include Configurable
+    class ValuesParser < Parser
       include TypeConverter
 
       config_param :keys, :string
       config_param :time_key, :string, :default => nil
       config_param :time_format, :string, :default => nil
 
-      attr_accessor :estimate_current_event
-
-      def initialize
-        super
-        @estimate_current_event = true
-      end
-
       def configure(conf)
         super
 
@@ -330,11 +334,16 @@ module Fluent
     class TSVParser < ValuesParser
       config_param :delimiter, :string, :default => "\t"
 
-      def call(text)
+      def configure(conf)
+        super
+        @key_num = @keys.length
+      end
+
+      def parse(text)
         if block_given?
-          yield values_map(text.split(@delimiter))
+          yield values_map(text.split(@delimiter, @key_num))
         else
-          return values_map(text.split(@delimiter))
+          return values_map(text.split(@delimiter, @key_num))
         end
       end
     end
@@ -349,7 +358,7 @@ module Fluent
         super(conf)
       end
 
-      def call(text)
+      def parse(text)
         @keys  = []
         values = []
 
@@ -373,7 +382,7 @@ module Fluent
         require 'csv'
       end
 
-      def call(text)
+      def parse(text)
         if block_given?
           yield values_map(CSV.parse_line(text))
         else
@@ -382,19 +391,10 @@ module Fluent
       end
     end
 
-    class NoneParser
-      include Configurable
-
+    class NoneParser < Parser
       config_param :message_key, :string, :default => 'message'
 
-      attr_accessor :estimate_current_event
-
-      def initialize
-        super
-        @estimate_current_event = true
-      end
-
-      def call(text)
+      def parse(text)
         record = {}
         record[@message_key] = text
         time = @estimate_current_event ? Engine.now : nil
@@ -406,13 +406,12 @@ module Fluent
       end
     end
 
-    class ApacheParser
-      include Configurable
-
+    class ApacheParser < Parser
       REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize
+        super
         @time_parser = TimeParser.new(TIME_FORMAT)
         @mutex = Mutex.new
       end
@@ -421,7 +420,7 @@ module Fluent
         {'format' => REGEXP, 'time_format' => TIME_FORMAT}
       end
 
-      def call(text)
+      def parse(text)
         m = REGEXP.match(text)
         unless m
           if block_given?
@@ -475,9 +474,7 @@ module Fluent
       end
     end
 
-    class SyslogParser
-      include Configurable
-
+    class SyslogParser < Parser
       # From existence TextParser pattern
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
@@ -486,11 +483,8 @@ module Fluent
       config_param :time_format, :string, :default => "%b %d %H:%M:%S"
       config_param :with_priority, :bool, :default => false
 
-      attr_accessor :estimate_current_event
-
       def initialize
         super
-        @estimate_current_event = true
         @mutex = Mutex.new
       end
 
@@ -505,7 +499,7 @@ module Fluent
         {'format' => @regexp, 'time_format' => @time_format}
       end
 
-      def call(text)
+      def parse(text)
         m = @regexp.match(text)
         unless m
           if block_given?
@@ -544,9 +538,7 @@ module Fluent
       end
     end
 
-    class MultilineParser
-      include Configurable
-
+    class MultilineParser < Parser
       config_param :format_firstline, :string, :default => nil
 
       FORMAT_MAX_NUM = 20
@@ -571,7 +563,7 @@ module Fluent
         end
       end
 
-      def call(text, &block)
+      def parse(text, &block)
         if block
           @parser.call(text, &block)
         else
@@ -716,9 +708,9 @@ module Fluent
 
     def parse(text, &block)
       if block
-        @parser.call(text, &block)
+        @parser.parse(text, &block)
       else # keep backward compatibility. Will be removed at v1
-        return @parser.call(text)
+        return @parser.parse(text)
       end
     end
   end