fluentd 0.10.50 → 0.10.51

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dcc68a7c0e54890fee8388d37e9801edac3fb77f
-  data.tar.gz: 992e8adb5d7c043f91f58e095aaf1d6cb5c7b363
+  metadata.gz: c0d7335ca94183ce7a33d7578b88addeb64b1c8f
+  data.tar.gz: 77807fdc964ab88d09ecc41787339c0825dba08c
 SHA512:
-  metadata.gz: 84b5ed1c244b405de5ed3d637fbad708cbb361a42a7efd13793c2992f7aad66c9c9d49256b105545e65e2627d5e3de64838e0df05d9e0d6bd82e6c47d7f56230
-  data.tar.gz: ad3c062f06ec7b472f7efb72acc693c9b71e83fc4eaf1df4c6c23ca5820543cb72bfea59778f3f5cb42f1510927f1f52c9f674a8bee3debdb3f86c4cc75e5950
+  metadata.gz: d3d90bcc098a60fc5525960445b25e6e5aea09f4221be6cbb943b8d656a34bb89ffdbfdad89a645394c85d953fd9577f2ec359f37f90c66520e658cb5c4f8d6a
+  data.tar.gz: a5759301599ea3243f1cb0021910ad411145cb524312d9d653be7bf94aeba6866eab42515c9dc77cb7279743f73290652c312feba6682113e57720b1951eca40

data/.travis.yml

@@ -5,7 +5,7 @@ rvm:
   - 2.0.0
   - 2.1
   - ruby-head
-  - rbx
+  - rbx-2
 
 os:
   - linux
@@ -24,4 +24,4 @@ script: bundle exec rake
 matrix:
   allow_failures:
     - rvm: ruby-head
-    - rvm: rbx
+    - rvm: rbx-2

data/CONTRIBUTING.md

@@ -0,0 +1,34 @@
+# Contributing to Fluentd
+
+We'd love your contribution. Here are the guidelines!
+
+## Got a question or problem?
+
+RESOURCES of [Official site](http://www.fluentd.org/) and [Fluentd documentation](http://docs.fluentd.org/) may help you.
+
+If you have further questions about Fluentd and plugins, please direct these to [Mailing List](https://groups.google.com/forum/#!forum/fluentd).
+
+Don't use Github issue for asking questions.
+Github issue is mainly for submitting a bug report or feature request. See below.
+
+## Found a bug?
+
+If you find a bug of Fluentd or a mistake in the documentation, you can help us by
+submitting an issue to Fluentd. Even better you can submit a Pull Request with a fix.
+
+* **Fluentd**: Use [fluentd](https://github.com/fluent/fluentd) repository.
+* **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs) repository.
+
+If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
+
+## Patch Guidelines
+
+Here are some things that would increase a chance that your patch is accepted:
+
+* Write tests.
+* Run tests before send Pull Request by `bundle exec rake test`
+* Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+
+There are some patches which are hard to write tests, e.g. process handling, concurrency issue or etc.
+In such case, please don't hesitate to submit a Pull Request.
+We can discuss how to manage a patch on Pull Request :)

data/ChangeLog

@@ -1,3 +1,14 @@
+Release 0.10.51 - 2014/07/04
+
+* in_tail: Fix rotation handling when new file not found
+* in_syslog: Change debug log level to warn at pattern mismatch
+* formatter: Use localtime by default
+* parser: Add SyslogParser
+* parser: Add "estimate_current_event" attribute to TextParsers not to set current time into parse result
+* config: Add <system> section to set core parameters via configuration file
+* test: Support v1 configuration in TestDriver
+* Add udp and tcp input plugins
+
 Release 0.10.50 - 2014/06/17
 
 * in_tail: Fix race condition at shutdown

data/README.md

@@ -22,11 +22,11 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
 
 ## More Information
 
-- Web site:  http://fluentd.org/
-- Documents: http://docs.fluentd.org/
+- Website: http://fluentd.org/
+- Documentation: http://docs.fluentd.org/
 - Source repository: http://github.com/fluent
 - Discussion: http://groups.google.com/group/fluentd
-- News Letters: http://go.treasuredata.com/Fluentd_education
+- Newsletters: http://get.treasuredata.com/Fluentd_education
 - Author: Sadayuki Furuhashi
 - Copyright: (c) 2011 FURUHASHI Sadayuki
 - License: Apache License, Version 2.0

data/Rakefile

@@ -11,6 +11,9 @@ task :test => [:base_test, :spec]
 
 desc 'Run test_unit based test'
 Rake::TestTask.new(:base_test) do |t|
+  # To run test for only one file (or file path pattern)
+  #  $ bundle exec rake base_test TEST=test/test_specified_path.rb
+  #  $ bundle exec rake base_test TEST=test/test_*.rb
   t.libs << "test"
   t.test_files = (Dir["test/test_*.rb"] + Dir["test/plugin/test_*.rb"] - ["helper.rb"]).sort
   t.verbose = true

data/fluent.conf

@@ -38,6 +38,7 @@
 # Listen DRb for debug
 <source>
   type debug_agent
+  bind 127.0.0.1
   port 24230
 </source>
 

data/lib/fluent/config.rb

@@ -20,23 +20,18 @@ module Fluent
   require 'fluent/config/element'
 
   module Config
-    def self.read(path, v1_config = false)
-      if v1_config
-        require 'fluent/config/v1_parser'
-        V1Parser.read(path)
-      else
-        require 'fluent/config/parser'
-        Parser.read(path)
-      end
-    end
-
     def self.parse(str, fname, basepath = Dir.pwd, v1_config = false)
-      if v1_config
-        require 'fluent/config/v1_parser'
-        V1Parser.parse(str, fname, basepath, Kernel.binding)
+      if fname =~ /\.rb$/
+        require 'fluent/config/dsl'
+        Config::DSL::Parser.parse(str, File.join(basepath, fname))
       else
-        require 'fluent/config/parser'
-        Parser.parse(str, fname, basepath)
+        if v1_config
+          require 'fluent/config/v1_parser'
+          V1Parser.parse(str, fname, basepath, Kernel.binding)
+        else
+          require 'fluent/config/parser'
+          Parser.parse(str, fname, basepath)
+        end
       end
     end
 

data/lib/fluent/config/parser.rb

@@ -4,13 +4,6 @@ module Fluent
     require 'fluent/config/element'
 
     class Parser
-      def self.read(path)
-        path = File.expand_path(path)
-        File.open(path) { |io|
-          parse(io, File.basename(path), File.dirname(path))
-        }
-      end
-
       def self.parse(io, fname, basepath = Dir.pwd)
         attrs, elems = Parser.new(basepath, io.each_line, fname).parse!(true)
         Element.new('ROOT', '', attrs, elems)

data/lib/fluent/config/v1_parser.rb

@@ -24,12 +24,6 @@ module Fluent
     class V1Parser < LiteralParser
       ELEMENT_NAME = /[a-zA-Z0-9_]+/
 
-      def self.read(path, eval_context = nil)
-        path = File.expand_path(path)
-        data = File.read(path)
-        parse(data, File.basename(path), File.dirname(path), eval_context)
-      end
-
       def self.parse(data, fname, basepath = Dir.pwd, eval_context = nil)
         ss = StringScanner.new(data)
         ps = V1Parser.new(ss, basepath, fname, eval_context)

data/lib/fluent/engine.rb

@@ -62,23 +62,19 @@ module Fluent
       @suppress_config_dump = flag
     end
 
-    def read_config(path)
-      $log.info "reading config file", :path=>path
-      File.open(path) {|io|
-        parse_config(io, File.basename(path), File.dirname(path))
-      }
+    def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
+      if fname =~ /\.rb$/
+        require 'fluent/config/dsl'
+        Config::DSL::Parser.parse(io, File.join(basepath, fname))
+      else
+        Config.parse(io, fname, basepath, v1_config)
+      end
     end
 
-    def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
-      conf = if fname =~ /\.rb$/
-               require 'fluent/config/dsl'
-               Config::DSL::Parser.parse(io, File.join(basepath, fname))
-             else
-               Config.parse(io, fname, basepath, v1_config)
-             end
+    def run_configure(conf)
       configure(conf)
-      conf.check_not_fetched {|key,e|
-        $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
+      conf.check_not_fetched { |key, e|
+        $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used." unless e.name == 'system'
       }
     end
 

data/lib/fluent/formatter.rb

@@ -27,6 +27,7 @@ module Fluent
           config_param :time_format, :string, :default => nil
           config_param :include_tag_key, :bool, :default => false
           config_param :tag_key, :string, :default => 'tag'
+          config_param :localtime, :bool, :default => true
         }
       end
 
@@ -35,8 +36,6 @@ module Fluent
 
         if conf['utc']
           @localtime = false
-        elsif conf['localtime']
-          @localtime = true
         end
         @timef = TimeFormatter.new(@time_format, @localtime)
       end

data/lib/fluent/log.rb

@@ -43,6 +43,18 @@ module Fluent
 
     LEVEL_TEXT = %w(trace debug info warn error fatal)
 
+    def self.str_to_level(log_level_str)
+      case log_level_str.downcase
+      when "trace" then LEVEL_TRACE
+      when "debug" then LEVEL_DEBUG
+      when "info"  then LEVEL_INFO
+      when "warn"  then LEVEL_WARN
+      when "error" then LEVEL_ERROR
+      when "fatal" then LEVEL_FATAL
+      else raise "Unknown log level: level = #{log_level_str}"
+      end
+    end
+
     def initialize(out=STDERR, level=LEVEL_TRACE, opts={})
       @out = out
       @level = level
@@ -313,15 +325,7 @@ module Fluent
     end
 
     def level=(log_level_str)
-      @level = case log_level_str.downcase
-               when "trace" then LEVEL_TRACE
-               when "debug" then LEVEL_DEBUG
-               when "info"  then LEVEL_INFO
-               when "warn"  then LEVEL_WARN
-               when "error" then LEVEL_ERROR
-               when "fatal" then LEVEL_FATAL
-               else raise "Unknown log level: level = #{log_level_str}"
-               end
+      @level = Log.str_to_level(log_level_str)
     end
 
     alias orig_enable_color enable_color

data/lib/fluent/parser.rb

@@ -128,6 +128,10 @@ module Fluent
 
       config_param :time_format, :string, :default => nil
 
+      # SET false BEFORE CONFIGURE, to return nil when time not parsed
+      # 'configure()' may raise errors for unexpected configurations
+      attr_accessor :estimate_current_event
+
       def initialize(regexp, conf={})
         super()
         @regexp = regexp
@@ -136,6 +140,7 @@ module Fluent
         end
 
         @time_parser = TimeParser.new(@time_format)
+        @estimate_current_event = true
         @mutex = Mutex.new
       end
 
@@ -144,6 +149,10 @@ module Fluent
         @time_parser = TimeParser.new(@time_format)
       end
 
+      def patterns
+        {'format' => @regexp, 'time_format' => @time_format}
+      end
+
       def call(text)
         m = @regexp.match(text)
         unless m
@@ -173,7 +182,9 @@ module Fluent
           end
         }
 
-        time ||= Engine.now
+        if @estimate_current_event
+          time ||= Engine.now
+        end
 
         if block_given?
           yield time, record
@@ -189,6 +200,15 @@ module Fluent
       config_param :time_key, :string, :default => 'time'
       config_param :time_format, :string, :default => nil
 
+      # SET false BEFORE CONFIGURE, to return nil when time not parsed
+      # 'configure()' may raise errors for unexpected configurations
+      attr_accessor :estimate_current_event
+
+      def initialize
+        super
+        @estimate_current_event = true
+      end
+
       def configure(conf)
         super
 
@@ -208,7 +228,11 @@ module Fluent
             time = value.to_i
           end
         else
-          time = Engine.now
+          if @estimate_current_event
+            time = Engine.now
+          else
+            time = nil
+          end
         end
 
         if block_given?
@@ -233,12 +257,21 @@ module Fluent
       config_param :time_key, :string, :default => nil
       config_param :time_format, :string, :default => nil
 
+      # SET false BEFORE CONFIGURE, to return nil when time not parsed
+      # 'configure()' may raise errors for unexpected configurations
+      attr_accessor :estimate_current_event
+
+      def initialize
+        super
+        @estimate_current_event = true
+      end
+
       def configure(conf)
         super
 
         @keys = @keys.split(",")
 
-        if @time_key && !@keys.include?(@time_key)
+        if @time_key && !@keys.include?(@time_key) && @estimate_current_event
           raise ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
         end
 
@@ -256,12 +289,18 @@ module Fluent
         if @time_key
           value = record.delete(@time_key)
           time = if value.nil?
-                   Engine.now
+                   if @estimate_current_event
+                     Engine.now
+                   else
+                     nil
+                   end
                  else
                    @mutex.synchronize { @time_parser.parse(value) }
                  end
-        else
+        elsif @estimate_current_event
           time = Engine.now
+        else
+          time = nil
         end
 
         convert_field_type!(record) if @type_converters
@@ -340,13 +379,21 @@ module Fluent
 
       config_param :message_key, :string, :default => 'message'
 
+      attr_accessor :estimate_current_event
+
+      def initialize
+        super
+        @estimate_current_event = true
+      end
+
       def call(text)
         record = {}
         record[@message_key] = text
+        time = @estimate_current_event ? Engine.now : nil
         if block_given?
-          yield Engine.now, record
+          yield time, record
         else
-          return Engine.now, record
+          return time, record
         end
       end
     end
@@ -355,12 +402,17 @@ module Fluent
       include Configurable
 
       REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+      TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize
-        @time_parser = TimeParser.new("%d/%b/%Y:%H:%M:%S %z")
+        @time_parser = TimeParser.new(TIME_FORMAT)
         @mutex = Mutex.new
       end
 
+      def patterns
+        {'format' => REGEXP, 'time_format' => TIME_FORMAT}
+      end
+
       def call(text)
         m = REGEXP.match(text)
         unless m
@@ -415,6 +467,75 @@ module Fluent
       end
     end
 
+    class SyslogParser
+      include Configurable
+
+      # From existence TextParser pattern
+      REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/
+      # From in_syslog default pattern
+      REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/
+      TIME_FORMAT = "%b %d %H:%M:%S"
+
+      config_param :with_priority, :bool, :default => false
+
+      attr_accessor :estimate_current_event
+
+      def initialize
+        super
+        @estimate_current_event = true
+        @time_parser = TextParser::TimeParser.new(TIME_FORMAT)
+        @mutex = Mutex.new
+      end
+
+      def configure(conf)
+        super
+
+        @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+      end
+
+      def patterns
+        {'format' => @regexp, 'time_format' => TIME_FORMAT}
+      end
+
+      def call(text)
+        m = @regexp.match(text)
+        unless m
+          if block_given?
+            yield nil, nil
+            return
+          else
+            return nil, nil
+          end
+        end
+
+        time = nil
+        record = {}
+
+        m.names.each { |name|
+          if value = m[name]
+            case name
+            when "pri"
+              record['pri'] = value.to_i
+            when "time"
+              time = @mutex.synchronize { @time_parser.parse(value.gsub(/ +/, ' ')) }
+            else
+              record[name] = value
+            end
+          end
+        }
+
+        if @estimate_current_event
+          time ||= Engine.now
+        end
+
+        if block_given?
+          yield time, record
+        else
+          return time, record
+        end
+      end
+    end
+
     class MultilineParser
       include Configurable
 
@@ -504,7 +625,7 @@ module Fluent
     {
       'apache' => Proc.new { RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
       'apache2' => Proc.new { ApacheParser.new },
-      'syslog' => Proc.new { RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}) },
+      'syslog' => Proc.new { SyslogParser.new },
       'json' => Proc.new { JSONParser.new },
       'tsv' => Proc.new { TSVParser.new },
       'ltsv' => Proc.new { LabeledTSVParser.new },